Refactored processing logic for openshift usage scripts

openshift_metrics/invoice.py

@@ -1,7 +1,7 @@
 import math
 from dataclasses import dataclass, field
 from collections import namedtuple
-from typing import List, Tuple, Optional
+from typing import List, Tuple
 from decimal import Decimal, ROUND_HALF_UP
 import datetime
 
@@ -121,7 +121,8 @@ def get_service_unit(self, su_definitions) -> ServiceUnit:
         return ServiceUnit(su_type, su_count, determining_resource)
 
     def get_runtime(
-        self, ignore_times: List[Tuple[datetime.datetime, datetime.datetime]] = None
+        self,
+        ignore_times: List[Tuple[datetime.datetime, datetime.datetime]] | None = None,
     ) -> Decimal:
         """Return runtime eligible for billing in hours"""
 
@@ -209,7 +210,9 @@ class ProjectInvoce:
     project_id: str
     rates: Rates
     su_definitions: dict
-    ignore_hours: Optional[List[Tuple[datetime.datetime, datetime.datetime]]] = None
+    ignore_hours: List[Tuple[datetime.datetime, datetime.datetime]] = field(
+        default_factory=list
+    )
     su_hours: dict = field(
         default_factory=lambda: {
             SU_CPU: 0,

openshift_metrics/merge.py

@@ -106,22 +106,12 @@ def load_metrics_metadata(files: List[str]) -> MetricsMetadata:
 
 
 def load_and_merge_metrics(interval_minutes, files: List[str]) -> MetricsProcessor:
-    """Load and merge metrics
-
-    Loads metrics from provided json files and then returns a processor
-    that has all the merged data.
-    """
+    """Load metrics from new-format files (namespaces + segments)."""
     processor = MetricsProcessor(interval_minutes)
     for file in files:
         with open(file, "r") as jsonfile:
             metrics_from_file = json.load(jsonfile)
-            cpu_request_metrics = metrics_from_file["cpu_metrics"]
-            memory_request_metrics = metrics_from_file["memory_metrics"]
-            gpu_request_metrics = metrics_from_file.get("gpu_metrics", None)
-            processor.merge_metrics("cpu_request", cpu_request_metrics)
-            processor.merge_metrics("memory_request", memory_request_metrics)
-            if gpu_request_metrics is not None:
-                processor.merge_metrics("gpu_request", gpu_request_metrics)
+            processor.load_segment_data(metrics_from_file["namespaces"])
     logger.info(f"Total metric files read: {len(files)}")
     return processor
 
@@ -181,18 +171,22 @@ def get_rates_and_outages(
         logger.info("Using nerc rates for rates and outages")
         rates_data = rates.load_from_url()
         invoice_rates = invoice.Rates(
-            cpu=rates_data.get_value_at("CPU SU Rate", meta.report_month, Decimal),
-            gpu_a100=rates_data.get_value_at(
-                "GPUA100 SU Rate", meta.report_month, Decimal
+            cpu=Decimal(
+                rates_data.get_value_at("CPU SU Rate", meta.report_month, Decimal)
+            ),
+            gpu_a100=Decimal(
+                rates_data.get_value_at("GPUA100 SU Rate", meta.report_month, Decimal)
             ),
-            gpu_a100sxm4=rates_data.get_value_at(
-                "GPUA100SXM4 SU Rate", meta.report_month, Decimal
+            gpu_a100sxm4=Decimal(
+                rates_data.get_value_at(
+                    "GPUA100SXM4 SU Rate", meta.report_month, Decimal
+                )
             ),
-            gpu_v100=rates_data.get_value_at(
-                "GPUV100 SU Rate", meta.report_month, Decimal
+            gpu_v100=Decimal(
+                rates_data.get_value_at("GPUV100 SU Rate", meta.report_month, Decimal)
             ),
-            gpu_h100=rates_data.get_value_at(
-                "GPUH100 SU Rate", meta.report_month, Decimal
+            gpu_h100=Decimal(
+                rates_data.get_value_at("GPUH100 SU Rate", meta.report_month, Decimal)
             ),
         )
         outage_data = outages.load_from_url()
@@ -303,11 +297,8 @@ def main():
         f"Generating report from {metrics_metadata.start_time_utc} to {metrics_metadata.end_time_utc + timedelta(days=1)} for {cluster_name}"
     )
 
-    # load and merge the metrics from the files, followed by condensing the metrics.
     processor = load_and_merge_metrics(metrics_metadata.interval_minutes, files)
-    condensed_metrics_dict = processor.condense_metrics(
-        ["cpu_request", "memory_request", "gpu_request", "gpu_type"]
-    )
+    condensed_metrics_dict = processor.merged_data
 
     # gather invoice rates and su defitions.
     invoice_rates, ignore_hours = get_rates_and_outages(args, metrics_metadata)