diff --git a/changelog.d/23994_aggregate_transform.enhancement.md b/changelog.d/23994_aggregate_transform.enhancement.md new file mode 100644 index 0000000000000..0f824c6fe28e4 --- /dev/null +++ b/changelog.d/23994_aggregate_transform.enhancement.md @@ -0,0 +1,3 @@ +The aggregate transform now supports aggregating metrics into Distribution metric types, enabling statistical analysis of metric values over time windows. + +authors: jlambatl diff --git a/src/transforms/aggregate/config.rs b/src/transforms/aggregate/config.rs index 4bf8ea69235ce..9f9860461cc65 100644 --- a/src/transforms/aggregate/config.rs +++ b/src/transforms/aggregate/config.rs @@ -138,6 +138,14 @@ pub enum AggregationMode { /// Stdev value of absolute metric; incremental metrics pass through unchanged. Stdev, + + /// Collects the values of absolute metrics into a single sorted distribution; incremental + /// metrics pass through unchanged. + /// + /// Gauges and sets (numeric members only) contribute one sample per value, distributions + /// contribute their existing samples, and aggregated histograms contribute one sample at each + /// bucket's upper limit per count in that bucket. Other absolute metric types are ignored. + Distribution, } const fn default_mode() -> AggregationMode { diff --git a/src/transforms/aggregate/event_time.rs b/src/transforms/aggregate/event_time.rs index 4d76ce326e9af..83e0696e8ba42 100644 --- a/src/transforms/aggregate/event_time.rs +++ b/src/transforms/aggregate/event_time.rs @@ -217,8 +217,9 @@ impl Aggregate { let bucket = self.event_time_buckets.entry(bucket_key).or_default(); Self::record_comparison_in_map(bucket, series, data, metadata, mode); } - AggregationMode::Mean | AggregationMode::Stdev => { - // `will_be_stored` has already guaranteed Absolute + Gauge. + AggregationMode::Mean | AggregationMode::Stdev | AggregationMode::Distribution => { + // `will_be_stored` has already guaranteed Absolute (+ Gauge for Mean/Stdev, or a + // convertible value for Distribution). // // Mean/Stdev write samples into `event_time_multi_buckets`; // an empty entry is also placed in `event_time_buckets` so @@ -466,6 +467,17 @@ impl Aggregate { continue; } + if matches!(self.config.mode, AggregationMode::Distribution) { + if let Some((data, metadata)) = + Self::distribution_from_entries(&entries) + { + output.push(Event::Metric(Metric::from_parts( + series, data, metadata, + ))); + } + continue; + } + let (mut final_sum, mut final_metadata) = entries.first().unwrap().clone(); for (data, metadata) in entries.iter().skip(1) { if !final_sum.update(data) { diff --git a/src/transforms/aggregate/tests.rs b/src/transforms/aggregate/tests.rs index 38e51b3cca2cd..0576e05080a9c 100644 --- a/src/transforms/aggregate/tests.rs +++ b/src/transforms/aggregate/tests.rs @@ -17,7 +17,7 @@ use crate::{ event::{ BatchNotifier, BatchStatus, BatchStatusReceiver, DatadogMetricOriginMetadata, Event, EventMetadata, EventStatus, Metric, - metric::{MetricKind, MetricValue}, + metric::{Bucket, MetricKind, MetricValue, Sample, StatisticKind}, }, schema::Definition, test_util::components::assert_transform_compliance, @@ -136,6 +136,23 @@ fn assert_gauge(event: &Event, expected: f64) { } } +/// Asserts `event` is a `Distribution` with exactly `expected_samples` (in order) and the given statistic. +fn assert_distribution_samples( + event: &Event, + expected_samples: &[f64], + expected_statistic: StatisticKind, +) { + let MetricValue::Distribution { samples, statistic } = event.as_metric().value() else { + panic!("expected Distribution, got {:?}", event.as_metric().value()); + }; + assert_eq!(*statistic, expected_statistic); + let values: Vec = samples.iter().map(|s| s.value).collect(); + assert_eq!( + values, expected_samples, + "samples should match expected values and be sorted" + ); +} + /// Deliver output events and assert every batch receiver resolves to Delivered. fn deliver_and_assert_batches(out: Vec, receivers: &mut [&mut BatchStatusReceiver]) { for receiver in receivers.iter_mut() { @@ -1737,3 +1754,548 @@ fn event_time_count_counts_mixed_kinds_in_same_bucket() { assert_eq!(out.len(), 1); assert_counter(&out[0], 2.0); } + +#[test] +fn absolute_distribution() { + let mut agg = Aggregate::new(&system_time_config(AggregationMode::Distribution)).unwrap(); + + let gauge_a_1 = make_metric( + "gauge_a", + MetricKind::Absolute, + MetricValue::Gauge { value: 25.0 }, + ); + let gauge_a_2 = make_metric( + "gauge_a", + MetricKind::Absolute, + MetricValue::Gauge { value: 30.0 }, + ); + let gauge_a_3 = make_metric( + "gauge_a", + MetricKind::Absolute, + MetricValue::Gauge { value: 35.0 }, + ); + let gauge_a_4 = make_metric( + "gauge_a", + MetricKind::Absolute, + MetricValue::Gauge { value: 36.0 }, + ); + let gauge_a_5 = make_metric( + "gauge_a", + MetricKind::Absolute, + MetricValue::Gauge { value: 37.0 }, + ); + let gauge_a_6 = make_metric( + "gauge_a", + MetricKind::Absolute, + MetricValue::Gauge { value: 38.0 }, + ); + let gauge_a_7 = make_metric( + "gauge_a", + MetricKind::Absolute, + MetricValue::Gauge { value: 39.0 }, + ); + let counter_a_1 = make_metric( + "counter_a", + MetricKind::Incremental, + MetricValue::Counter { value: 110.0 }, + ); + // Record seven gauge values in non-sorted order to verify sorting + agg.record(gauge_a_4); // 36.0 + agg.record(gauge_a_1); // 25.0 + agg.record(gauge_a_7); // 39.0 + agg.record(gauge_a_2); // 30.0 + agg.record(gauge_a_5); // 37.0 + agg.record(gauge_a_3); // 35.0 + agg.record(gauge_a_6); // 38.0 + // Verify that incremental metrics are ignored + agg.record(counter_a_1); + + let mut out = vec![]; + agg.flush_into(&mut out); + assert_eq!(1, out.len()); + + // Verify it's a distribution with 7 samples, properly sorted + assert_distribution_samples( + &out[0], + &[25.0, 30.0, 35.0, 36.0, 37.0, 38.0, 39.0], + StatisticKind::Histogram, + ); +} + +#[test] +fn absolute_distribution_distributions() { + let mut agg = Aggregate::new(&system_time_config(AggregationMode::Distribution)).unwrap(); + + let distribution_a_1 = make_metric( + "dist_a", + MetricKind::Absolute, + MetricValue::Distribution { + samples: vec![35.0, 25.0, 36.0, 30.0] // Intentionally unsorted + .into_iter() + .map(|v| Sample { value: v, rate: 1 }) + .collect(), + statistic: StatisticKind::Summary, + }, + ); + let distribution_a_2 = make_metric( + "dist_a", + MetricKind::Absolute, + MetricValue::Distribution { + samples: vec![39.0, 37.0, 38.0] // Intentionally unsorted + .into_iter() + .map(|v| Sample { value: v, rate: 1 }) + .collect(), + statistic: StatisticKind::Summary, + }, + ); + let distribution_a_3 = make_metric( + "dist_a", + MetricKind::Absolute, + MetricValue::Distribution { + samples: vec![456.2, 99.0, 113.0] // Intentionally unsorted + .into_iter() + .map(|v| Sample { value: v, rate: 1 }) + .collect(), + statistic: StatisticKind::Summary, + }, + ); + // Record three Distribution values + agg.record(distribution_a_1); + agg.record(distribution_a_2); + agg.record(distribution_a_3); + + let mut out = vec![]; + agg.flush_into(&mut out); + assert_eq!(1, out.len()); + + // Verify it's a distribution with 10 samples, properly sorted + assert_distribution_samples( + &out[0], + &[25.0, 30.0, 35.0, 36.0, 37.0, 38.0, 39.0, 99.0, 113.0, 456.2], + StatisticKind::Histogram, + ); +} + +#[test] +fn distribution_histogram() { + let mut agg = Aggregate::new(&system_time_config(AggregationMode::Distribution)).unwrap(); + + let histogram_a_1 = make_metric( + "histogram_a", + MetricKind::Absolute, + MetricValue::AggregatedHistogram { + count: 5, + sum: 18.0, + buckets: vec![ + Bucket { + upper_limit: 1.0, + count: 1, + }, + Bucket { + upper_limit: 2.0, + count: 2, + }, + Bucket { + upper_limit: 5.0, + count: 1, + }, + Bucket { + upper_limit: 10.0, + count: 1, + }, + ], + }, + ); + let histogram_a_2 = make_metric( + "histogram_a", + MetricKind::Absolute, + MetricValue::AggregatedHistogram { + count: 5, + sum: 18.0, + buckets: vec![ + Bucket { + upper_limit: 1.0, + count: 1, + }, + Bucket { + upper_limit: 2.0, + count: 2, + }, + Bucket { + upper_limit: 5.0, + count: 1, + }, + Bucket { + upper_limit: 10.0, + count: 1, + }, + ], + }, + ); + let histogram_a_3 = make_metric( + "histogram_a", + MetricKind::Absolute, + MetricValue::AggregatedHistogram { + count: 5, + sum: 18.0, + buckets: vec![ + Bucket { + upper_limit: 1.0, + count: 1, + }, + Bucket { + upper_limit: 2.0, + count: 2, + }, + Bucket { + upper_limit: 5.0, + count: 1, + }, + Bucket { + upper_limit: 10.0, + count: 1, + }, + ], + }, + ); + + let expected_vals = vec![ + 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 5.0, 5.0, 5.0, 10.0, 10.0, 10.0, + ]; + + // Record Metrics Values + agg.record(histogram_a_1); + agg.record(histogram_a_2); + agg.record(histogram_a_3); + + let mut out = vec![]; + agg.flush_into(&mut out); + assert_eq!(1, out.len()); + + // Verify it's a distribution with 15 samples + assert_distribution_samples(&out[0], &expected_vals, StatisticKind::Histogram); +} + +#[test] +fn distribution_set() { + let mut agg = Aggregate::new(&system_time_config(AggregationMode::Distribution)).unwrap(); + + let set_a_1 = make_metric( + "set_a", + MetricKind::Absolute, + MetricValue::Set { + values: vec![ + "39".into(), + "25".into(), + "99".into(), + "35".into(), + "113".into(), + "30".into(), + "456.2".into(), + "36".into(), + "37".into(), + "38".into(), + ] + .into_iter() + .collect(), + }, + ); + + let set_a_2 = make_metric( + "set_a", + MetricKind::Absolute, + MetricValue::Set { + values: vec![ + "40".into(), + "26".into(), + "100".into(), + "36".into(), + "114".into(), + "31".into(), + "556.2".into(), + "37".into(), + "38".into(), + "39".into(), + ] + .into_iter() + .collect(), + }, + ); + // Record two set values + agg.record(set_a_1); + agg.record(set_a_2); + + let mut out = vec![]; + agg.flush_into(&mut out); + assert_eq!( + 1, + out.len(), + "expected 1 output metric but got {}", + out.len() + ); + + // Verify it's a distribution with 20 samples (10 from each set), properly sorted + let expected = vec![ + 25.0, 26.0, 30.0, 31.0, 35.0, 36.0, 36.0, 37.0, 37.0, 38.0, 38.0, 39.0, 39.0, 40.0, 99.0, + 100.0, 113.0, 114.0, 456.2, 556.2, + ]; + assert_distribution_samples(&out[0], &expected, StatisticKind::Histogram); +} + +#[test] +fn distribution_mixed_types() { + let mut agg = Aggregate::new(&system_time_config(AggregationMode::Distribution)).unwrap(); + + // Test that different metric series can each become distributions + let gauge_a = make_metric( + "gauge_a", + MetricKind::Absolute, + MetricValue::Gauge { value: 10.0 }, + ); + + let distribution_b = make_metric( + "dist_b", + MetricKind::Absolute, + MetricValue::Distribution { + samples: vec![20.0, 30.0] + .into_iter() + .map(|v| Sample { value: v, rate: 1 }) + .collect(), + statistic: StatisticKind::Summary, + }, + ); + + let histogram_c = make_metric( + "histogram_c", + MetricKind::Absolute, + MetricValue::AggregatedHistogram { + count: 2, + sum: 90.0, + buckets: vec![ + Bucket { + upper_limit: 40.0, + count: 1, + }, + Bucket { + upper_limit: 50.0, + count: 1, + }, + ], + }, + ); + + agg.record(gauge_a); + agg.record(distribution_b); + agg.record(histogram_c); + + let mut out = vec![]; + agg.flush_into(&mut out); + assert_eq!(3, out.len()); // Each different series produces its own distribution + + // Verify all outputs are distributions + for event in out { + if let MetricValue::Distribution { samples, statistic } = event.as_metric().value() { + assert_eq!(*statistic, StatisticKind::Histogram); + assert!(!samples.is_empty()); + } else { + panic!("Expected Distribution metric value"); + } + } +} + +#[test] +fn distribution_set_with_invalid_values() { + let mut agg = Aggregate::new(&system_time_config(AggregationMode::Distribution)).unwrap(); + + let set_with_invalid = make_metric( + "set_a", + MetricKind::Absolute, + MetricValue::Set { + values: vec![ + "10.0".into(), + "not_a_number".into(), // Should be skipped + "20.0".into(), + "invalid".into(), // Should be skipped + "30.0".into(), + ] + .into_iter() + .collect(), + }, + ); + + agg.record(set_with_invalid); + + let mut out = vec![]; + agg.flush_into(&mut out); + assert_eq!(1, out.len()); + + // Only the 3 valid numeric values should be included + assert_distribution_samples(&out[0], &[10.0, 20.0, 30.0], StatisticKind::Histogram); +} + +#[test] +fn distribution_multiple_series() { + let mut agg = Aggregate::new(&system_time_config(AggregationMode::Distribution)).unwrap(); + + let gauge_a = make_metric( + "gauge_a", + MetricKind::Absolute, + MetricValue::Gauge { value: 10.0 }, + ); + let gauge_b = make_metric( + "gauge_b", + MetricKind::Absolute, + MetricValue::Gauge { value: 20.0 }, + ); + + agg.record(gauge_a); + agg.record(gauge_b); + + let mut out = vec![]; + agg.flush_into(&mut out); + + // Should produce 2 separate distributions + assert_eq!(2, out.len()); + + for event in out { + if let MetricValue::Distribution { samples, .. } = event.as_metric().value() { + assert_eq!(samples.len(), 1); + } else { + panic!("Expected Distribution metric value"); + } + } +} + +#[test] +fn distribution_incremental_passes_through_unchanged() { + let mut agg = Aggregate::new(&system_time_config(AggregationMode::Distribution)).unwrap(); + + let passthrough = agg.record(make_metric( + "counter_a", + MetricKind::Incremental, + MetricValue::Counter { value: 110.0 }, + )); + assert_counter( + &passthrough.expect("incremental metrics pass through"), + 110.0, + ); + + let mut out = vec![]; + agg.flush_into(&mut out); + assert!( + out.is_empty(), + "passed-through metrics must not be aggregated" + ); +} + +#[test] +fn distribution_ignores_absolute_values_without_samples() { + let mut agg = Aggregate::new(&system_time_config(AggregationMode::Distribution)).unwrap(); + + // Absolute counters cannot contribute samples: not aggregated, not passed through. + assert!( + agg.record(make_metric( + "counter_a", + MetricKind::Absolute, + MetricValue::Counter { value: 5.0 }, + )) + .is_none() + ); + // A set with no numeric members yields no samples either. + let mut values = BTreeSet::new(); + values.insert("not-a-number".to_string()); + assert!( + agg.record(make_metric( + "set_a", + MetricKind::Absolute, + MetricValue::Set { values }, + )) + .is_none() + ); + + let mut out = vec![]; + agg.flush_into(&mut out); + assert!(out.is_empty(), "no empty distribution should be emitted"); +} + +#[test] +fn event_time_distribution_buckets_and_passes_through() { + let interval_ms = 10_000; + let mut agg = Aggregate::new(&event_time_config_with( + interval_ms, + AggregationMode::Distribution, + |et| et.max_future_ms = 600_000, + )) + .unwrap(); + + // Incremental metrics pass through without creating buckets or touching the watermark. + let passthrough = agg.record(make_metric( + "passthrough", + MetricKind::Incremental, + MetricValue::Counter { value: 1.0 }, + )); + assert!(passthrough.is_some()); + assert!(agg.event_time_buckets.is_empty()); + assert!(agg.event_time_multi_buckets.is_empty()); + + // Absolute values that cannot yield samples must not materialise a bucket. + assert!( + agg.record(make_metric_with_timestamp( + "ignored", + MetricKind::Absolute, + MetricValue::Counter { value: 1.0 }, + event_time_bucket_timestamp(interval_ms, 1), + )) + .is_none() + ); + assert!(agg.event_time_buckets.is_empty()); + assert!(agg.event_time_multi_buckets.is_empty()); + + // Mixed convertible types in the same bucket merge into one sorted distribution. + let ts = event_time_bucket_timestamp(interval_ms, 1); + agg.record(make_metric_with_timestamp( + "mixed", + MetricKind::Absolute, + MetricValue::Gauge { value: 3.0 }, + ts, + )); + agg.record(make_metric_with_timestamp( + "mixed", + MetricKind::Absolute, + MetricValue::Distribution { + samples: vec![ + Sample { + value: 1.0, + rate: 1, + }, + Sample { + value: 2.0, + rate: 1, + }, + ], + statistic: StatisticKind::Histogram, + }, + ts, + )); + agg.record(make_metric_with_timestamp( + "mixed", + MetricKind::Absolute, + MetricValue::AggregatedHistogram { + buckets: vec![Bucket { + upper_limit: 4.0, + count: 2, + }], + count: 2, + sum: 8.0, + }, + ts, + )); + + let out = flush_final(&mut agg); + assert_eq!(out.len(), 1); + assert_distribution_samples( + &out[0], + &[1.0, 2.0, 3.0, 4.0, 4.0], + StatisticKind::Histogram, + ); +} diff --git a/src/transforms/aggregate/transform.rs b/src/transforms/aggregate/transform.rs index e6e1803f6bff7..a6de216f049fe 100644 --- a/src/transforms/aggregate/transform.rs +++ b/src/transforms/aggregate/transform.rs @@ -10,7 +10,7 @@ use async_stream::stream; use futures::{Stream, StreamExt}; use vector_lib::event::{ MetricValue, - metric::{Metric, MetricData, MetricKind, MetricSeries}, + metric::{Metric, MetricData, MetricKind, MetricSeries, Sample, StatisticKind}, }; use crate::{ @@ -54,6 +54,11 @@ enum InnerMode { Stdev { multi_map: HashMap>, }, + + /// Collects absolute metrics into a distribution; incremental metrics pass through unchanged. + Distribution { + multi_map: HashMap>, + }, } impl From for InnerMode { @@ -74,6 +79,9 @@ impl From for InnerMode { AggregationMode::Stdev => InnerMode::Stdev { multi_map: HashMap::default(), }, + AggregationMode::Distribution => InnerMode::Distribution { + multi_map: HashMap::default(), + }, } } } @@ -163,7 +171,8 @@ impl Aggregate { | InnerMode::Max | InnerMode::Min | InnerMode::Mean { .. } - | InnerMode::Stdev { .. }, + | InnerMode::Stdev { .. } + | InnerMode::Distribution { .. }, MetricKind::Incremental, ) => { return Some(Event::Metric(Metric::from_parts(series, data, metadata))); @@ -196,6 +205,11 @@ impl Aggregate { } } } + (InnerMode::Distribution { multi_map }, MetricKind::Absolute) => { + if Self::distribution_accepts(&data.value) { + multi_map.entry(series).or_default().push((data, metadata)); + } + } } emit!(AggregateEventRecorded); @@ -214,18 +228,38 @@ impl Aggregate { | AggregationMode::Max | AggregationMode::Min | AggregationMode::Mean - | AggregationMode::Stdev, + | AggregationMode::Stdev + | AggregationMode::Distribution, MetricKind::Incremental ) ) } /// Returns `true` for metrics that system-time accepts but does not store or - /// pass through — currently absolute non-gauge values in `Mean`/`Stdev`. + /// pass through — absolute non-gauge values in `Mean`/`Stdev`, and absolute values that + /// cannot contribute samples in `Distribution`. pub(crate) const fn is_silently_ignored(mode: AggregationMode, data: &MetricData) -> bool { - matches!(mode, AggregationMode::Mean | AggregationMode::Stdev) - && matches!(data.kind, MetricKind::Absolute) - && !matches!(data.value, MetricValue::Gauge { value: _ }) + if !matches!(data.kind, MetricKind::Absolute) { + return false; + } + match mode { + AggregationMode::Mean | AggregationMode::Stdev => { + !matches!(data.value, MetricValue::Gauge { value: _ }) + } + AggregationMode::Distribution => !Self::distribution_accepts(&data.value), + _ => false, + } + } + + /// Returns `true` if `Distribution` mode can turn this value into samples. + pub(crate) const fn distribution_accepts(value: &MetricValue) -> bool { + matches!( + value, + MetricValue::Gauge { .. } + | MetricValue::Set { .. } + | MetricValue::Distribution { .. } + | MetricValue::AggregatedHistogram { .. } + ) } /// Returns `true` iff a record with the given `kind`/`value` would be @@ -261,6 +295,10 @@ impl Aggregate { matches!(data.kind, MetricKind::Absolute) && matches!(data.value, MetricValue::Gauge { value: _ }) } + // `Distribution` only records absolute values it can convert to samples. + AggregationMode::Distribution => { + matches!(data.kind, MetricKind::Absolute) && Self::distribution_accepts(&data.value) + } } } @@ -378,9 +416,9 @@ impl Aggregate { } let multi_map = match &mut self.mode { - InnerMode::Mean { multi_map } | InnerMode::Stdev { multi_map } => { - std::mem::take(multi_map) - } + InnerMode::Mean { multi_map } + | InnerMode::Stdev { multi_map } + | InnerMode::Distribution { multi_map } => std::mem::take(multi_map), _ => HashMap::default(), }; @@ -389,6 +427,15 @@ impl Aggregate { continue; } + // Distribution builds its output from the raw entries; `update()` only + // works for same-typed metrics and would reject mixed gauge/set/histogram input. + if matches!(self.mode, InnerMode::Distribution { .. }) { + if let Some((data, metadata)) = Self::distribution_from_entries(&entries) { + output.push(Event::Metric(Metric::from_parts(series, data, metadata))); + } + continue; + } + let (mut final_sum, mut final_metadata) = entries.first().unwrap().clone(); for (data, metadata) in entries.iter().skip(1) { if !final_sum.update(data) { @@ -444,6 +491,78 @@ impl Aggregate { } } +impl Aggregate { + /// Builds a single sorted `Distribution` from absolute entries of mixed types, merging + /// their metadata. + /// + /// - `Gauge`: one sample. + /// - `Set`: one sample per member that parses as `f64`; others are skipped. + /// - `Distribution`: existing samples are merged. + /// - `AggregatedHistogram`: one sample at each bucket's upper limit per count in the bucket. + /// + /// Returns `None` when no entry yields a sample, so no empty distribution is emitted. + pub(crate) fn distribution_from_entries( + entries: &[MetricEntry], + ) -> Option<(MetricData, EventMetadata)> { + let mut samples = Vec::new(); + + for (data, _) in entries { + match data.value() { + MetricValue::Gauge { value } => samples.push(Sample { + value: *value, + rate: 1, + }), + MetricValue::Set { values } => { + samples.extend(values.iter().filter_map(|value| { + value + .parse::() + .ok() + .map(|value| Sample { value, rate: 1 }) + })); + } + MetricValue::Distribution { + samples: existing, .. + } => samples.extend(existing.iter().copied()), + MetricValue::AggregatedHistogram { buckets, .. } => { + for bucket in buckets { + for _ in 0..bucket.count { + samples.push(Sample { + value: bucket.upper_limit, + rate: 1, + }); + } + } + } + _ => (), + } + } + + if samples.is_empty() { + return None; + } + samples.sort_by(|a, b| a.value.total_cmp(&b.value)); + + let (first_data, first_metadata) = entries.first()?; + let metadata = + entries + .iter() + .skip(1) + .fold(first_metadata.clone(), |mut acc, (_, metadata)| { + acc.merge(metadata.clone()); + acc + }); + let data = MetricData { + time: first_data.time, + kind: MetricKind::Absolute, + value: MetricValue::Distribution { + samples, + statistic: StatisticKind::Histogram, + }, + }; + Some((data, metadata)) + } +} + impl TaskTransform for Aggregate { fn transform( mut self: Box, diff --git a/website/cue/reference/components/transforms/aggregate.cue b/website/cue/reference/components/transforms/aggregate.cue index 45b8dac83ba4e..a34a4debee007 100644 --- a/website/cue/reference/components/transforms/aggregate.cue +++ b/website/cue/reference/components/transforms/aggregate.cue @@ -173,7 +173,10 @@ components: transforms: aggregate: { aggregated into a single `incremental` `counter` with a value of 23. Two `absolute` `gauge` metrics with values 93 and 95 would result in a single `absolute` `gauge` with the value of 95. More complex types like `distribution`, `histogram`, `set`, and `summary` behave similarly with `incremental` - values being combined in a manner that makes sense based on their type. + values being combined in a manner that makes sense based on their type. When using `distribution` as the + aggregation mode, the values of `absolute` metrics (`gauge`, `set`, `distribution`, and `histogram`) + are collected and combined into a single sorted `distribution`, and `incremental` metrics pass through + unchanged. """ } diff --git a/website/cue/reference/components/transforms/generated/aggregate.cue b/website/cue/reference/components/transforms/generated/aggregate.cue index 6eeb7593ebd13..daeae6fed096a 100644 --- a/website/cue/reference/components/transforms/generated/aggregate.cue +++ b/website/cue/reference/components/transforms/generated/aggregate.cue @@ -85,9 +85,17 @@ generated: components: transforms: aggregate: configuration: { type: string: { default: "Auto" enum: { - Auto: "Default mode. Sums incremental metrics and uses the latest value for absolute metrics." - Count: "Counts metrics for incremental and absolute metrics" - Diff: "Returns difference between latest value for absolute; incremental metrics pass through unchanged." + Auto: "Default mode. Sums incremental metrics and uses the latest value for absolute metrics." + Count: "Counts metrics for incremental and absolute metrics" + Diff: "Returns difference between latest value for absolute; incremental metrics pass through unchanged." + Distribution: """ + Collects the values of absolute metrics into a single sorted distribution; incremental + metrics pass through unchanged. + + Gauges and sets (numeric members only) contribute one sample per value, distributions + contribute their existing samples, and aggregated histograms contribute one sample at each + bucket's upper limit per count in that bucket. Other absolute metric types are ignored. + """ Latest: "Returns the latest value for absolute metrics; incremental metrics pass through unchanged." Max: "Max value of absolute metric; incremental metrics pass through unchanged." Mean: "Mean value of absolute metric; incremental metrics pass through unchanged."