fix: make Azure billing usage and forecast windows configurable

changelog/fragments/1784719246-azure-billing-configurable-lookback-window.yaml

@@ -0,0 +1,50 @@
+# REQUIRED
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# REQUIRED for all kinds
+# Change summary; a 80ish characters long description of the change.
+summary: Add configurable usage lookback and forecast windows to the Azure billing metricset
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+description: |
+  Adds `billing_usage_lookback` and `billing_forecast_window` duration config options to the
+  Azure billing metricset. `billing_usage_lookback` (default `24h`) allows re-querying multiple
+  previous days of usage data so that Azure's documented up-to-72h cost corrections are captured.
+  `billing_forecast_window` (default `720h`) controls the length of the forecast period. Defaults
+  preserve the existing behaviour.
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# impact:
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# action:
+
+# REQUIRED for all kinds
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: metricbeat
+
+# AUTOMATED
+# OPTIONAL to manually add other PR URLs
+# PR URL: A link the PR that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+# pr: https://github.com/owner/repo/1234
+
+# AUTOMATED
+# OPTIONAL to manually add other issue URLs
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+# issue: https://github.com/owner/repo/1234

docs/reference/metricbeat/metricbeat-metricset-azure-billing.md

@@ -39,6 +39,15 @@ This metricset allows users to retrieve usage details and forecast information o
 
 If none of the 2 options are entered then the subscription ID will be used as scope.
 
+`billing_usage_lookback`
+:   (*duration*, default: `24h`) How far back to query usage data on each fetch. Azure Cost Management
+    data is documented to be updated for up to 72 hours after a billing period closes, so increasing
+    this to `72h` ensures that late cost corrections from Azure are re-queried and ingested.
+
+`billing_forecast_window`
+:   (*duration*, default: `720h`) The length of the forecast period, counted forward from the forecast
+    start date (reference time minus 2 days). Defaults to 30 days (`720h`).
+
 ## Fields [_fields]
 
 For a description of each field in the metricset, see the [exported fields](/reference/metricbeat/exported-fields-azure.md) section.

docs/reference/metricbeat/metricbeat-module-azure.md

@@ -258,6 +258,13 @@ metricbeat.modules:
   client_secret: '${AZURE_CLIENT_SECRET:""}'
   tenant_id: '${AZURE_TENANT_ID:""}'
   subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+  # billing_usage_lookback controls how far back usage data is queried on each fetch.
+  # Defaults to 24h (previous full day). Azure Cost Management data can be updated for up
+  # to 72 hours after a billing period closes, so setting this to 72h ensures late
+  # corrections are captured.
+  #billing_usage_lookback: 72h
+  # billing_forecast_window controls the length of the forecast period. Defaults to 720h (30 days).
+  #billing_forecast_window: 720h
 
 - module: azure
   metricsets:

x-pack/metricbeat/metricbeat.reference.yml

@@ -400,6 +400,13 @@ metricbeat.modules:
   client_secret: '${AZURE_CLIENT_SECRET:""}'
   tenant_id: '${AZURE_TENANT_ID:""}'
   subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+  # billing_usage_lookback controls how far back usage data is queried on each fetch.
+  # Defaults to 24h (previous full day). Azure Cost Management data can be updated for up
+  # to 72 hours after a billing period closes, so setting this to 72h ensures late
+  # corrections are captured.
+  #billing_usage_lookback: 72h
+  # billing_forecast_window controls the length of the forecast period. Defaults to 720h (30 days).
+  #billing_forecast_window: 720h
 
 - module: azure
   metricsets:

x-pack/metricbeat/module/azure/_meta/config.reference.yml

@@ -92,6 +92,13 @@
   client_secret: '${AZURE_CLIENT_SECRET:""}'
   tenant_id: '${AZURE_TENANT_ID:""}'
   subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+  # billing_usage_lookback controls how far back usage data is queried on each fetch.
+  # Defaults to 24h (previous full day). Azure Cost Management data can be updated for up
+  # to 72 hours after a billing period closes, so setting this to 72h ensures late
+  # corrections are captured.
+  #billing_usage_lookback: 72h
+  # billing_forecast_window controls the length of the forecast period. Defaults to 720h (30 days).
+  #billing_forecast_window: 720h
 
 - module: azure
   metricsets:

x-pack/metricbeat/module/azure/_meta/config.yml

@@ -101,6 +101,11 @@
 #  tenant_id: '${AZURE_TENANT_ID:""}'
 #  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
 #  refresh_list_interval: 600s
+#  # How far back to query usage data on each fetch (default 24h). Azure Cost Management data
+#  # can be updated for up to 72 hours, so set this to 72h to capture late cost corrections.
+#  billing_usage_lookback: 72h
+#  # Length of the forecast period (default 720h / 30 days).
+#  billing_forecast_window: 720h
 
 #- module: azure
 #  metricsets:

x-pack/metricbeat/module/azure/billing/_meta/docs.md

@@ -26,3 +26,12 @@ This metricset allows users to retrieve usage details and forecast information o
 :   (*string*) Retrieve usage details based on the billing account ID scope.
 
 If none of the 2 options are entered then the subscription ID will be used as scope.
+
+`billing_usage_lookback`
+:   (*duration*, default: `24h`) How far back to query usage data on each fetch. Azure Cost Management
+    data is documented to be updated for up to 72 hours after a billing period closes, so increasing
+    this to `72h` ensures that late cost corrections from Azure are re-queried and ingested.
+
+`billing_forecast_window`
+:   (*duration*, default: `720h`) The length of the forecast period, counted forward from the forecast
+    start date (reference time minus 2 days). Defaults to 30 days (`720h`).

x-pack/metricbeat/module/azure/billing/billing.go

@@ -16,6 +16,16 @@ import (
 	"github.com/elastic/elastic-agent-libs/logp"
 )
 
+const (
+	// defaultUsageLookback is the usage query window when billing_usage_lookback is unset.
+	// It matches the original hardcoded behaviour: query the single previous full UTC day.
+	defaultUsageLookback = 24 * time.Hour
+
+	// defaultForecastWindow is the forecast query window when billing_forecast_window is unset.
+	// It matches the original hardcoded behaviour: 30 days forward from the forecast start date.
+	defaultForecastWindow = 30 * 24 * time.Hour
+)
+
 // init registers the MetricSet with the central registry as soon as the program
 // starts. The New function will be called later to instantiate an instance of
 // the MetricSet for each host defined in the module's configuration. After the
@@ -42,9 +52,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	if err != nil {
 		return nil, fmt.Errorf("error unpack raw module config using UnpackConfig: %w", err)
 	}
-	if err != nil {
-		return nil, err
-	}
+	applyBillingDefaults(&config)
 	// instantiate monitor client
 	billingClient, err := NewClient(config, base.Logger())
 	if err != nil {
@@ -57,6 +65,17 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	}, nil
 }
 
+// applyBillingDefaults fills in default values for billing-specific duration
+// config fields that were not explicitly set by the user.
+func applyBillingDefaults(cfg *azure.Config) {
+	if cfg.BillingUsageLookback == 0 {
+		cfg.BillingUsageLookback = defaultUsageLookback
+	}
+	if cfg.BillingForecastWindow == 0 {
+		cfg.BillingForecastWindow = defaultForecastWindow
+	}
+}
+
 // TimeIntervalOptions represents the options used to retrieve the billing data.
 type TimeIntervalOptions struct {
 	// Usage details start time (UTC).
@@ -76,8 +95,8 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 	// reference time used to calculate usage and forecast time intervals.
 	referenceTime := time.Now()
 
-	usageStart, usageEnd := usageIntervalFrom(referenceTime)
-	forecastStart, forecastEnd := forecastIntervalFrom(referenceTime)
+	usageStart, usageEnd := usageIntervalFrom(referenceTime, m.client.Config.BillingUsageLookback)
+	forecastStart, forecastEnd := forecastIntervalFrom(referenceTime, m.client.Config.BillingForecastWindow)
 
 	timeIntervalOptions := TimeIntervalOptions{
 		usageStart:    usageStart,
@@ -110,31 +129,35 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 	return nil
 }
 
-// usageIntervalFrom returns the start/end times (UTC) of the usage period given the `reference` time.
+// usageIntervalFrom returns the start/end times (UTC) of the usage period given the
+// reference time and a lookback duration.
 //
-// Currently, the usage period is the start/end time (00:00:00->23:59:59 UTC) of the day before the reference time.
+// The usage period ends at start-of-today minus one second (i.e. yesterday 23:59:59 UTC)
+// and starts lookback duration before that.
 //
-// For example, if the reference time is 2007-01-09 09:41:00Z, the usage period is:
+// For example, with reference 2007-01-09 09:41:00Z and a 24h lookback, the usage period is:
 //
 //	2007-01-08 00:00:00Z -> 2007-01-08 23:59:59Z
-func usageIntervalFrom(reference time.Time) (time.Time, time.Time) {
-	beginningOfDay := reference.UTC().Truncate(24 * time.Hour).Add((-24) * time.Hour)
-	endOfDay := beginningOfDay.Add(time.Hour * 24).Add(time.Second * (-1))
-	return beginningOfDay, endOfDay
+//
+// With a 72h lookback it covers the three previous full days:
+//
+//	2007-01-06 00:00:00Z -> 2007-01-08 23:59:59Z
+func usageIntervalFrom(reference time.Time, lookback time.Duration) (time.Time, time.Time) {
+	startOfToday := reference.UTC().Truncate(24 * time.Hour)
+	return startOfToday.Add(-lookback), startOfToday.Add(-time.Second)
 }
 
-// forecastIntervalFrom returns the start/end times (UTC) of the forecast period, given the `reference` time.
+// forecastIntervalFrom returns the start/end times (UTC) of the forecast period given the
+// reference time and a window duration.
 //
-// Currently, the forecast period is the start/end times (00:00:00->23:59:59 UTC) of the current month relative to the
-// reference time.
+// The forecast period always starts at reference minus 2 days (00:00:00 UTC) and extends
+// forward for the given window duration.
 //
-// For example, if the reference time is 2007-01-09 09:41:00Z, the forecast period is:
-// The forecast data is fetched from current day - 2 and for next 30 days.
+// For example, with reference 2007-01-09 09:41:00Z and a 30-day (720h) window:
 //
-//	2007-01-07T00:00:00Z -> 2007-02-05:59:59Z
-func forecastIntervalFrom(reference time.Time) (time.Time, time.Time) {
-	referenceUTC := reference.UTC().Truncate(24 * time.Hour).Add((-48) * time.Hour)
-	forecastStartDate := time.Date(referenceUTC.Year(), referenceUTC.Month(), referenceUTC.Day(), 0, 0, 0, 0, time.UTC)
-	forecastEndDate := forecastStartDate.AddDate(0, 0, 0).Add(-1*time.Second).AddDate(0, 0, 30)
-	return forecastStartDate, forecastEndDate
+//	2007-01-07T00:00:00Z -> 2007-02-05T23:59:59Z
+func forecastIntervalFrom(reference time.Time, window time.Duration) (time.Time, time.Time) {
+	forecastStart := reference.UTC().Truncate(24 * time.Hour).Add(-48 * time.Hour)
+	forecastEnd := forecastStart.Add(window).Add(-time.Second)
+	return forecastStart, forecastEnd
 }

x-pack/metricbeat/module/azure/billing/billing_test.go

@@ -11,6 +11,8 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/elastic/beats/v7/x-pack/metricbeat/module/azure"
 )
 
 func TestUsagePeriodFrom(t *testing.T) {
@@ -22,7 +24,23 @@ func TestUsagePeriodFrom(t *testing.T) {
 		expectedEndTime, err := time.Parse("2006-01-02 15:04:05", "2007-01-08 23:59:59")
 		assert.NoError(t, err)
 
-		actualStartTime, actualEndTime := usageIntervalFrom(referenceTime)
+		actualStartTime, actualEndTime := usageIntervalFrom(referenceTime, 24*time.Hour)
+
+		assert.Equal(t, expectedStartTime, actualStartTime)
+		assert.Equal(t, expectedEndTime, actualEndTime)
+	})
+
+	t.Run("widens the window when lookback is 72h", func(t *testing.T) {
+		referenceTime, err := time.Parse("2006-01-02 15:04:05", "2007-01-09 09:41:00")
+		assert.NoError(t, err)
+		// 72h lookback from start-of-today (2007-01-09 00:00:00) → 2007-01-06 00:00:00
+		// end always = start-of-today minus one second → 2007-01-08 23:59:59
+		expectedStartTime, err := time.Parse("2006-01-02 15:04:05", "2007-01-06 00:00:00")
+		assert.NoError(t, err)
+		expectedEndTime, err := time.Parse("2006-01-02 15:04:05", "2007-01-08 23:59:59")
+		assert.NoError(t, err)
+
+		actualStartTime, actualEndTime := usageIntervalFrom(referenceTime, 72*time.Hour)
 
 		assert.Equal(t, expectedStartTime, actualStartTime)
 		assert.Equal(t, expectedEndTime, actualEndTime)
@@ -39,9 +57,52 @@ func TestForecastPeriodFrom(t *testing.T) {
 		expectedEndTime, err := time.Parse("2006-01-02 15:04:05", "2007-02-05 23:59:59")
 		assert.NoError(t, err)
 
-		actualStartTime, actualEndTime := forecastIntervalFrom(referenceTime)
+		actualStartTime, actualEndTime := forecastIntervalFrom(referenceTime, 30*24*time.Hour)
+
+		assert.Equal(t, expectedStartTime, actualStartTime)
+		assert.Equal(t, expectedEndTime, actualEndTime)
+	})
+
+	t.Run("uses the configured window duration", func(t *testing.T) {
+		referenceTime, err := time.Parse("2006-01-02 15:04:05", "2007-01-09 09:41:00")
+		assert.NoError(t, err)
+
+		// forecast always starts at reference - 2 days; end = start + window - 1s
+		// with a 7-day window: 2007-01-07 00:00:00 + 7d - 1s = 2007-01-13 23:59:59
+		expectedStartTime, err := time.Parse("2006-01-02 15:04:05", "2007-01-07 00:00:00")
+		assert.NoError(t, err)
+		expectedEndTime, err := time.Parse("2006-01-02 15:04:05", "2007-01-13 23:59:59")
+		assert.NoError(t, err)
+
+		actualStartTime, actualEndTime := forecastIntervalFrom(referenceTime, 7*24*time.Hour)
 
 		assert.Equal(t, expectedStartTime, actualStartTime)
 		assert.Equal(t, expectedEndTime, actualEndTime)
 	})
 }
+
+func TestApplyBillingDefaults(t *testing.T) {
+	t.Run("sets usage lookback and forecast window when both are unset", func(t *testing.T) {
+		cfg := azure.Config{}
+		applyBillingDefaults(&cfg)
+
+		assert.Equal(t, defaultUsageLookback, cfg.BillingUsageLookback)
+		assert.Equal(t, defaultForecastWindow, cfg.BillingForecastWindow)
+	})
+
+	t.Run("does not override an explicitly configured usage lookback", func(t *testing.T) {
+		cfg := azure.Config{BillingUsageLookback: 72 * time.Hour}
+		applyBillingDefaults(&cfg)
+
+		assert.Equal(t, 72*time.Hour, cfg.BillingUsageLookback)
+		assert.Equal(t, defaultForecastWindow, cfg.BillingForecastWindow)
+	})
+
+	t.Run("does not override an explicitly configured forecast window", func(t *testing.T) {
+		cfg := azure.Config{BillingForecastWindow: 14 * 24 * time.Hour}
+		applyBillingDefaults(&cfg)
+
+		assert.Equal(t, defaultUsageLookback, cfg.BillingUsageLookback)
+		assert.Equal(t, 14*24*time.Hour, cfg.BillingForecastWindow)
+	})
+}

x-pack/metricbeat/module/azure/billing/client.go

@@ -71,7 +71,7 @@ func (client *Client) GetMetrics(timeOpts TimeIntervalOptions) (Usage, error) {
 		Infow("Getting usage details for scope")
 
 	filter := fmt.Sprintf(
-		"properties/usageStart eq '%s' and properties/usageEnd eq '%s'",
+		"properties/usageStart ge '%s' and properties/usageEnd le '%s'",
 		timeOpts.usageStart.Format(time.RFC3339Nano),
 		timeOpts.usageEnd.Format(time.RFC3339Nano),
 	)

x-pack/metricbeat/module/azure/billing/client_test.go

@@ -26,8 +26,8 @@ var (
 )
 
 func TestClient(t *testing.T) {
-	usageStart, usageEnd := usageIntervalFrom(time.Now())
-	forecastStart, forecastEnd := forecastIntervalFrom(time.Now())
+	usageStart, usageEnd := usageIntervalFrom(time.Now(), defaultUsageLookback)
+	forecastStart, forecastEnd := forecastIntervalFrom(time.Now(), defaultForecastWindow)
 	opts := TimeIntervalOptions{
 		usageStart:    usageStart,
 		usageEnd:      usageEnd,
@@ -65,3 +65,33 @@ func TestClient(t *testing.T) {
 		m.AssertExpectations(t)
 	})
 }
+
+func TestClientUsesRangeFilterForUsageDetails(t *testing.T) {
+	opts := TimeIntervalOptions{
+		usageStart:    time.Date(2026, 7, 19, 0, 0, 0, 0, time.UTC),
+		usageEnd:      time.Date(2026, 7, 21, 23, 59, 59, 0, time.UTC),
+		forecastStart: time.Date(2026, 7, 19, 0, 0, 0, 0, time.UTC),
+		forecastEnd:   time.Date(2026, 8, 17, 23, 59, 59, 0, time.UTC),
+	}
+
+	client := NewMockClient(logptest.NewTestingLogger(t, ""))
+	client.Config = azure.Config{SubscriptionId: "sub"}
+	m := &MockService{}
+
+	expectedFilter := "properties/usageStart ge '2026-07-19T00:00:00Z' and properties/usageEnd le '2026-07-21T23:59:59Z'"
+	m.On(
+		"GetUsageDetails",
+		"subscriptions/sub",
+		"properties/meterDetails",
+		expectedFilter,
+		armconsumption.MetrictypeActualCostMetricType,
+		"2026-07-19",
+		"2026-07-21",
+	).Return(armconsumption.UsageDetailsListResult{}, nil)
+	m.On("GetForecast", "subscriptions/sub", opts.forecastStart, opts.forecastEnd).Return(armcostmanagement.QueryResult{}, nil)
+	client.BillingService = m
+
+	_, err := client.GetMetrics(opts)
+	assert.NoError(t, err)
+	m.AssertExpectations(t)
+}