diff --git a/openshift_metrics/config.py b/openshift_metrics/config.py index 2c2ec3a..6f1ff4f 100644 --- a/openshift_metrics/config.py +++ b/openshift_metrics/config.py @@ -14,5 +14,19 @@ S3_SECRET_ACCESS_KEY = os.getenv("S3_OUTPUT_SECRET_ACCESS_KEY") S3_INVOICE_BUCKET = os.getenv("S3_INVOICE_BUCKET", "nerc-invoicing") S3_METRICS_BUCKET = os.getenv("S3_METRICS_BUCKET", "openshift_metrics") + +# Billing Configuration + +# Rate Configuration +USE_NERC_RATES = os.getenv("USE_NERC_RATES", "false").lower() == "true" +# Individual rates (used when USE_NERC_RATES=false) +RATE_CPU_SU = os.getenv("RATE_CPU_SU") +RATE_GPU_V100_SU = os.getenv("RATE_GPU_V100_SU") +RATE_GPU_A100SXM4_SU = os.getenv("RATE_GPU_A100SXM4_SU") +RATE_GPU_A100_SU = os.getenv("RATE_GPU_A100_SU") +RATE_GPU_H100_SU = os.getenv("RATE_GPU_H100_SU") + + +# Prometheus Query Configuration PROM_QUERY_INTERVAL_MINUTES = int(os.getenv("PROM_QUERY_INTERVAL_MINUTES", 15)) assert PROM_QUERY_INTERVAL_MINUTES >= 1, "Query interval must be at least 1 minute" diff --git a/openshift_metrics/merge.py b/openshift_metrics/merge.py index ea01980..ba6d4f9 100644 --- a/openshift_metrics/merge.py +++ b/openshift_metrics/merge.py @@ -13,7 +13,16 @@ from openshift_metrics import utils, invoice from openshift_metrics.metrics_processor import MetricsProcessor -from openshift_metrics.config import S3_INVOICE_BUCKET, PROM_QUERY_INTERVAL_MINUTES +from openshift_metrics.config import ( + S3_INVOICE_BUCKET, + USE_NERC_RATES, + RATE_CPU_SU, + RATE_GPU_V100_SU, + RATE_GPU_A100SXM4_SU, + RATE_GPU_A100_SU, + RATE_GPU_H100_SU, + PROM_QUERY_INTERVAL_MINUTES, +) logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -90,16 +99,6 @@ def main(): nargs="*", help="List of timestamp ranges in UTC to ignore in the format 'YYYY-MM-DDTHH:MM:SS,YYYY-MM-DDTHH:MM:SS'", ) - parser.add_argument( - "--use-nerc-rates", - action="store_true", - help="Use rates from the nerc-rates repo", - ) - parser.add_argument("--rate-cpu-su", type=Decimal) - parser.add_argument("--rate-gpu-v100-su", type=Decimal) - parser.add_argument("--rate-gpu-a100sxm4-su", type=Decimal) - parser.add_argument("--rate-gpu-a100-su", type=Decimal) - parser.add_argument("--rate-gpu-h100-su", type=Decimal) args = parser.parse_args() files = args.files @@ -165,7 +164,7 @@ def main(): datetime.strptime(report_start_date, "%Y-%m-%d"), "%Y-%m" ) - if args.use_nerc_rates: + if USE_NERC_RATES: logger.info("Using nerc rates for rates and outages") rates_data = rates.load_from_url() invoice_rates = invoice.Rates( @@ -182,12 +181,23 @@ def main(): report_start_date, report_end_date, cluster_name ) else: + if RATE_CPU_SU is None: + raise ValueError("RATE_CPU_SU environment variable must be set") + if RATE_GPU_V100_SU is None: + raise ValueError("RATE_GPU_V100_SU environment variable must be set") + if RATE_GPU_A100SXM4_SU is None: + raise ValueError("RATE_GPU_A100SXM4_SU environment variable must be set") + if RATE_GPU_A100_SU is None: + raise ValueError("RATE_GPU_A100_SU environment variable must be set") + if RATE_GPU_H100_SU is None: + raise ValueError("RATE_GPU_H100_SU environment variable must be set") + invoice_rates = invoice.Rates( - cpu=Decimal(args.rate_cpu_su), - gpu_a100=Decimal(args.rate_gpu_a100_su), - gpu_a100sxm4=Decimal(args.rate_gpu_a100sxm4_su), - gpu_v100=Decimal(args.rate_gpu_v100_su), - gpu_h100=Decimal(args.rate_gpu_h100_su), + cpu=Decimal(RATE_CPU_SU), + gpu_a100=Decimal(RATE_GPU_A100_SU), + gpu_a100sxm4=Decimal(RATE_GPU_A100SXM4_SU), + gpu_v100=Decimal(RATE_GPU_V100_SU), + gpu_h100=Decimal(RATE_GPU_H100_SU), ) ignore_hours = args.ignore_hours @@ -210,15 +220,19 @@ def main(): else: pod_report_file = f"Pod NERC OpenShift {report_month}.csv" - report_start_date = datetime.strptime(report_start_date, "%Y-%m-%d").replace( + report_start_date_dt = datetime.strptime(report_start_date, "%Y-%m-%d").replace( tzinfo=UTC ) - report_end_date = datetime.strptime(report_end_date, "%Y-%m-%d").replace(tzinfo=UTC) - - logger.info( - f"Generating report from {report_start_date} to {report_end_date + timedelta(days=1)} for {cluster_name}" + report_end_date_dt = datetime.strptime(report_end_date, "%Y-%m-%d").replace( + tzinfo=UTC ) + if report_start_date_dt.month != report_end_date_dt.month: + logger.warning("The report spans multiple months") + + report_start_date = report_start_date_dt + report_end_date = report_end_date_dt + condensed_metrics_dict = processor.condense_metrics( ["cpu_request", "memory_request", "gpu_request", "gpu_type"] )