feat(config): warn about ignored configuration fields

Cargo.toml

@@ -262,6 +262,7 @@ similar-asserts = "2.0.0"
 strum = { version = "0.28", features = ["derive"] }
 
 [dependencies]
+jsonschema.workspace = true
 cfg-if.workspace = true
 reqwest.workspace = true
 reqwest_13 = { package = "reqwest", version = "0.13", default-features = false, features = ["json", "native-tls", "rustls-no-provider"] }
@@ -550,7 +551,6 @@ criterion = { workspace = true, features = ["html_reports", "async_tokio"] }
 hyper-1 = { package = "hyper", version = "1", default-features = false, features = ["server"] }
 hyper-util = { version = "0.1", default-features = false, features = ["server-auto"] }
 itertools.workspace = true
-jsonschema.workspace = true
 libc.workspace = true
 mock_instant.workspace = true
 proptest.workspace = true

config/examples/es_s3_hybrid.yaml

@@ -38,4 +38,4 @@ sinks:
       codec: "json"
     compression: "gzip"
     batch:
-      max_size: 10000000 # 10mb uncompressed
+      max_bytes: 10000000 # 10mb uncompressed

deprecation.d/ignored-configuration-fields.md

@@ -0,0 +1,40 @@
+---
+what: "Ignored configuration fields"
+deprecated_since: "0.61.0"
+---
+
+<a id="ignored-configuration-fields"></a>
+
+Configuration fields that Vector accepts but discards during deserialization are deprecated.
+Vector now warns about these fields when loading configuration, including during `vector validate`.
+Existing configurations continue to work.
+
+Correct misspelled fields or remove fields that do not belong to the selected enum variant.
+For example, replace this HTTP sink request setting:
+
+```yaml
+request:
+  retry_atempts: 3
+```
+
+with:
+
+```yaml
+request:
+  retry_attempts: 3
+```
+
+For options that select an enum variant, keep only the fields used by that variant. For example,
+HTTP authentication with `strategy: bearer` uses `token`. Remove `username` and `password`, which
+belong to the `basic` variant:
+
+```yaml
+auth:
+  strategy: bearer
+  token: example-token
+```
+
+Warnings identify field paths without including their values. Maps that accept arbitrary keys,
+such as HTTP headers, remain supported. Recognized settings documented as inactive in a particular
+mode, such as `value_limit` with `mode: excluded` in `tag_cardinality_limit`, are not affected by this
+deprecation.

lib/vector-buffers/src/config.rs

@@ -222,6 +222,8 @@ pub enum BufferType {
     /// This is more performant, but less durable. Data will be lost if Vector is restarted
     /// forcefully or crashes.
     #[configurable(title = "Events are buffered in memory.")]
+    // BufferTypeVisitor defaults a missing type to memory.
+    #[configurable(metadata(serde::optional_tag = "type"))]
     Memory {
         /// The terms around how to express buffering limits, can be in size or `bytes_size`.
         #[serde(flatten)]

src/common/http/server_auth.rs

@@ -87,15 +87,15 @@ enum HttpServerAuthRepr {
         #[configurable(metadata(docs::examples = "my-secret-token"))]
         token: SensitiveString,
         /// Accepted but ignored for this strategy.
-        #[configurable(metadata(docs::hidden))]
+        #[configurable(metadata(docs::hidden, serde::ignored))]
         #[serde(default, skip_serializing_if = "String::is_empty")]
         username: String,
         /// Accepted but ignored for this strategy.
-        #[configurable(metadata(docs::hidden))]
+        #[configurable(metadata(docs::hidden, serde::ignored))]
         #[serde(default, skip_serializing_if = "String::is_empty")]
         password: String,
         /// Accepted but ignored for this strategy.
-        #[configurable(metadata(docs::hidden))]
+        #[configurable(metadata(docs::hidden, serde::ignored))]
         #[serde(default, skip_serializing_if = "String::is_empty")]
         source: String,
     },
@@ -106,15 +106,15 @@ enum HttpServerAuthRepr {
         /// The VRL boolean expression.
         source: String,
         /// Accepted but ignored for this strategy.
-        #[configurable(metadata(docs::hidden))]
+        #[configurable(metadata(docs::hidden, serde::ignored))]
         #[serde(default, skip_serializing_if = "String::is_empty")]
         username: String,
         /// Accepted but ignored for this strategy.
-        #[configurable(metadata(docs::hidden))]
+        #[configurable(metadata(docs::hidden, serde::ignored))]
         #[serde(default, skip_serializing_if = "String::is_empty")]
         password: String,
         /// Accepted but ignored for this strategy.
-        #[configurable(metadata(docs::hidden))]
+        #[configurable(metadata(docs::hidden, serde::ignored))]
         #[serde(default, skip_serializing_if = "String::is_empty")]
         token: String,
     },
@@ -138,11 +138,11 @@ enum HttpServerAuthRepr {
         #[configurable(metadata(docs::examples = "password"))]
         password: SensitiveString,
         /// Accepted but ignored for this strategy.
-        #[configurable(metadata(docs::hidden))]
+        #[configurable(metadata(docs::hidden, serde::ignored))]
         #[serde(default, skip_serializing_if = "String::is_empty")]
         token: String,
         /// Accepted but ignored for this strategy.
-        #[configurable(metadata(docs::hidden))]
+        #[configurable(metadata(docs::hidden, serde::ignored))]
         #[serde(default, skip_serializing_if = "String::is_empty")]
         source: String,
     },

src/config/builder.rs

@@ -76,6 +76,10 @@ pub struct ConfigBuilder {
     #[serde(default, skip)]
     #[doc(hidden)]
     pub allow_empty: bool,
+
+    /// Diagnostics collected while loading, before unused fields are discarded.
+    #[serde(skip)]
+    pub(crate) loading_warnings: Vec<String>,
 }
 
 impl From<Config> for ConfigBuilder {
@@ -127,6 +131,7 @@ impl From<Config> for ConfigBuilder {
             secret,
             graceful_shutdown_duration,
             allow_empty: false,
+            loading_warnings: Vec::new(),
         }
     }
 }
@@ -141,9 +146,7 @@ impl ConfigBuilder {
     pub fn build(self) -> Result<Config, Vec<String>> {
         let (config, warnings) = self.build_with_warnings()?;
 
-        for warning in warnings {
-            warn!("{warning}");
-        }
+        super::loading::log_warnings(&warnings);
 
         Ok(config)
     }
@@ -290,6 +293,7 @@ impl ConfigBuilder {
         self.transforms.extend(with.transforms);
         self.tests.extend(with.tests);
         self.secret.extend(with.secret);
+        self.loading_warnings.extend(with.loading_warnings);
 
         Ok(())
     }

src/config/compiler.rs

@@ -61,6 +61,7 @@ pub fn compile(mut builder: ConfigBuilder) -> Result<(Config, Vec<String>), Vec<
         secret,
         graceful_shutdown_duration,
         allow_empty: _,
+        loading_warnings: mut warnings,
     } = builder;
     let graph = match Graph::new(nodes, global.wildcard_matching.unwrap_or_default()) {
         Ok(graph) => graph,
@@ -140,7 +141,7 @@ pub fn compile(mut builder: ConfigBuilder) -> Result<(Config, Vec<String>), Vec<
 
         config.propagate_acknowledgements()?;
 
-        let warnings = validation::warnings(&config);
+        warnings.extend(validation::warnings(&config));
 
         Ok((config, warnings))
     } else {

src/config/loading/config_builder.rs

@@ -3,7 +3,7 @@ use std::{collections::HashMap, io::Read};
 use indexmap::IndexMap;
 
 use super::{
-    ComponentHint, Process, deserialize_config_map, loader, prepare_input,
+    ComponentHint, Process, deserialize_config_map, ignored_fields, loader, prepare_input,
     representation::ConfigMap, secret,
 };
 use crate::config::{
@@ -94,6 +94,7 @@ impl Process for ConfigBuilderLoader {
 
     /// Merge a configuration map with a `ConfigBuilder`. Component types extend specific keys.
     fn merge(&mut self, map: ConfigMap, hint: Option<ComponentHint>) -> Result<(), Vec<String>> {
+        let ignored = ignored_fields::find(&map, hint);
         match hint {
             Some(ComponentHint::Source) => {
                 self.builder
@@ -131,6 +132,9 @@ impl Process for ConfigBuilderLoader {
             }
         }
 
+        self.builder
+            .loading_warnings
+            .extend(ignored_fields::warnings(&ignored));
         Ok(())
     }
 }