feat(config): Support loading secrets from directories

This change allows loading secrets from files in directories. In
particular, this is useful for loading secrets from systemd credentials
directory (see https://systemd.io/CREDENTIALS) and similar mechanisms.

src/config/loading/secret.rs

@@ -27,6 +27,8 @@ use crate::{
 // - "SECRET[backend..secret.name]" will match and capture "backend" and ".secret.name"
 // - "SECRET[secret_name]" will not match
 // - "SECRET[.secret.name]" will not match
+//
+// Note that `directory` backend assumes that secret key name does not contain path separator.
 pub static COLLECTOR: Lazy<Regex> =
     Lazy::new(|| Regex::new(r"SECRET\[([[:word:]]+)\.([[:word:].]+)\]").unwrap());
 

src/secrets/directory.rs

@@ -0,0 +1,54 @@
+use std::collections::{HashMap, HashSet};
+use std::path::PathBuf;
+
+use vector_lib::configurable::{component::GenerateConfig, configurable_component};
+
+use crate::{config::SecretBackend, signal};
+
+/// Configuration for the `directory` secrets backend.
+#[configurable_component(secrets("directory"))]
+#[derive(Clone, Debug)]
+pub struct DirectoryBackend {
+    /// Directory path to read secrets from.
+    pub path: PathBuf,
+
+    /// Remove trailing whitespace from file contents.
+    #[serde(default)]
+    pub remove_trailing_whitespace: bool,
+}
+
+impl GenerateConfig for DirectoryBackend {
+    fn generate_config() -> toml::Value {
+        toml::Value::try_from(DirectoryBackend {
+            path: PathBuf::from("/path/to/secrets"),
+            remove_trailing_whitespace: false,
+        })
+        .unwrap()
+    }
+}
+
+impl SecretBackend for DirectoryBackend {
+    async fn retrieve(
+        &mut self,
+        secret_keys: HashSet<String>,
+        _: &mut signal::SignalRx,
+    ) -> crate::Result<HashMap<String, String>> {
+        let mut secrets = HashMap::new();
+        for k in secret_keys.into_iter() {
+            // NB secret key name cannot contain path separator, so we do not access files outside
+            // of the configured directory path.
+            let path = self.path.join(&k);
+            let contents = tokio::fs::read_to_string(&path).await?;
+            let secret = if self.remove_trailing_whitespace {
+                contents.trim_end()
+            } else {
+                &contents
+            };
+            if secret.is_empty() {
+                return Err(format!("secret in file '{}' was empty", k).into());
+            }
+            secrets.insert(k, secret.to_string());
+        }
+        Ok(secrets)
+    }
+}

src/secrets/mod.rs

@@ -10,6 +10,7 @@ use crate::{config::SecretBackend, signal};
 mod aws_secrets_manager;
 mod exec;
 mod file;
+mod directory;
 mod test;
 
 /// Configurable secret backends in Vector.
@@ -22,6 +23,9 @@ pub enum SecretBackends {
     /// File.
     File(file::FileBackend),
 
+    /// Directory.
+    Directory(directory::DirectoryBackend),
+
     /// Exec.
     Exec(exec::ExecBackend),
 
@@ -39,6 +43,7 @@ impl NamedComponent for SecretBackends {
     fn get_component_name(&self) -> &'static str {
         match self {
             Self::File(config) => config.get_component_name(),
+            Self::Directory(config) => config.get_component_name(),
             Self::Exec(config) => config.get_component_name(),
             #[cfg(feature = "secrets-aws-secrets-manager")]
             Self::AwsSecretsManager(config) => config.get_component_name(),

tests/behavior/config/directory-secrets/jkl

@@ -0,0 +1 @@
+jkl.retrieved

tests/behavior/config/secret.toml

@@ -10,13 +10,19 @@
   type = "file"
   path = "tests/behavior/config/file-secrets.json"
 
+[secret.directory_backend]
+  type = "directory"
+  path = "tests/behavior/config/directory-secrets"
+  remove_trailing_whitespace = true
+
 [transforms.add_field_from_secret]
   inputs = []
   type = "remap"
   source = '''
   .foobar = "SECRET[test_backend.abc]"
   .foobarbaz = "SECRET[exec_backend.def]"
   .foobarbazqux = "SECRET[file_backend.ghi]"
+  .foobarbazquxquux = "SECRET[directory_backend.jkl]"
   '''
 
 [[tests]]
@@ -33,4 +39,5 @@
       .foobar == "this_is_a_secret_value"
       .foobarbaz == "def.retrieved"
       .foobarbazqux == "ghi.retrieved"
+      .foobarbazquxquux == "jkl.retrieved"
       '''

website/cue/reference/configuration.cue

@@ -508,6 +508,40 @@ configuration: {
 						}
 					}
 				}
+				directory: {
+					required: true
+					description: """
+						Retrieve secrets from file contents in a directory.
+
+						The directory must contain files with names corresponding to secret keys.
+
+						If an error occurrs while reading the file, Vector will log the error and exit.
+
+						Secrets are loaded when Vector starts or if Vector receives a `SIGHUP` signal triggering its
+						configuration reload process.
+						"""
+					type: object: options: {
+						path: {
+							description: """
+								The path of the directory with secrets.
+								"""
+							required: true
+							type: string: {
+								examples: [
+									"${CREDENTIALS_DIRECTORY}", // https://systemd.io/CREDENTIALS
+									"/path/to/secrets-directory",
+								]
+							}
+						}
+						remove_trailing_whitespace: {
+							description: """
+								Remove trailing whitespace from file contents.
+								"""
+							required:    false
+							type: bool: default: false
+						}
+					}
+				}
 				exec: {
 					required: true
 					description: """