feat(config): support loading secrets from files and directories (#21282

)

* feat(config): Support loading secrets from files

* feat(config): Support loading secrets from directories

This change allows loading secrets from files in directories. In
particular, this is useful for loading secrets from systemd credentials
directory (see https://systemd.io/CREDENTIALS) and similar mechanisms.

* chore: Update spelling dictionary

* chore: Add changelog entry for file/directory secrets backends

* fix: Move test data from tests/behavior to tests/data directory

.github/actions/spelling/expect.txt

@@ -388,6 +388,8 @@ Fomichev
 fonttbl
 foob
 foobarbaz
+foobarbazqux
+foobarbazquxquux
 foobarfoobarfoo
 FOOBARy
 foobaz

changelog.d/add-file-secrets.enhancement.md

@@ -0,0 +1,4 @@
+Vector now supports two additional back-ends for loading secrets: `file`, for reading a set of
+secrets from a JSON file, and `directory`, for loading secrets from a list of files.
+
+authors: tie

src/secrets/directory.rs

@@ -0,0 +1,52 @@
+use std::collections::{HashMap, HashSet};
+use std::path::PathBuf;
+
+use vector_lib::configurable::{component::GenerateConfig, configurable_component};
+
+use crate::{config::SecretBackend, signal};
+
+/// Configuration for the `directory` secrets backend.
+#[configurable_component(secrets("directory"))]
+#[derive(Clone, Debug)]
+pub struct DirectoryBackend {
+    /// Directory path to read secrets from.
+    pub path: PathBuf,
+
+    /// Remove trailing whitespace from file contents.
+    #[serde(default)]
+    pub remove_trailing_whitespace: bool,
+}
+
+impl GenerateConfig for DirectoryBackend {
+    fn generate_config() -> toml::Value {
+        toml::Value::try_from(DirectoryBackend {
+            path: PathBuf::from("/path/to/secrets"),
+            remove_trailing_whitespace: false,
+        })
+        .unwrap()
+    }
+}
+
+impl SecretBackend for DirectoryBackend {
+    async fn retrieve(
+        &mut self,
+        secret_keys: HashSet<String>,
+        _: &mut signal::SignalRx,
+    ) -> crate::Result<HashMap<String, String>> {
+        let mut secrets = HashMap::new();
+        for k in secret_keys.into_iter() {
+            let file_path = self.path.join(&k);
+            let contents = tokio::fs::read_to_string(&file_path).await?;
+            let secret = if self.remove_trailing_whitespace {
+                contents.trim_end()
+            } else {
+                &contents
+            };
+            if secret.is_empty() {
+                return Err(format!("secret in file '{}' was empty", k).into());
+            }
+            secrets.insert(k, secret.to_string());
+        }
+        Ok(secrets)
+    }
+}

src/secrets/file.rs

@@ -0,0 +1,46 @@
+use std::collections::{HashMap, HashSet};
+use std::path::PathBuf;
+
+use vector_lib::configurable::{component::GenerateConfig, configurable_component};
+
+use crate::{config::SecretBackend, signal};
+
+/// Configuration for the `file` secrets backend.
+#[configurable_component(secrets("file"))]
+#[derive(Clone, Debug)]
+pub struct FileBackend {
+    /// File path to read secrets from.
+    pub path: PathBuf,
+}
+
+impl GenerateConfig for FileBackend {
+    fn generate_config() -> toml::Value {
+        toml::Value::try_from(FileBackend {
+            path: PathBuf::from("/path/to/secret"),
+        })
+        .unwrap()
+    }
+}
+
+impl SecretBackend for FileBackend {
+    async fn retrieve(
+        &mut self,
+        secret_keys: HashSet<String>,
+        _: &mut signal::SignalRx,
+    ) -> crate::Result<HashMap<String, String>> {
+        let contents = tokio::fs::read_to_string(&self.path).await?;
+        let output = serde_json::from_str::<HashMap<String, String>>(&contents)?;
+        let mut secrets = HashMap::new();
+        for k in secret_keys.into_iter() {
+            if let Some(secret) = output.get(&k) {
+                if secret.is_empty() {
+                    return Err(format!("secret for key '{}' was empty", k).into());
+                }
+                secrets.insert(k, secret.to_string());
+            } else {
+                return Err(format!("secret for key '{}' was not retrieved", k).into());
+            }
+        }
+        Ok(secrets)
+    }
+}

src/secrets/mod.rs

@@ -8,7 +8,9 @@ use crate::{config::SecretBackend, signal};
 
 #[cfg(feature = "secrets-aws-secrets-manager")]
 mod aws_secrets_manager;
+mod directory;
 mod exec;
+mod file;
 mod test;
 
 /// Configurable secret backends in Vector.
@@ -18,6 +20,12 @@ mod test;
 #[enum_dispatch(SecretBackend)]
 #[serde(tag = "type", rename_all = "snake_case")]
 pub enum SecretBackends {
+    /// File.
+    File(file::FileBackend),
+
+    /// Directory.
+    Directory(directory::DirectoryBackend),
+
     /// Exec.
     Exec(exec::ExecBackend),
 
@@ -34,6 +42,8 @@ pub enum SecretBackends {
 impl NamedComponent for SecretBackends {
     fn get_component_name(&self) -> &'static str {
         match self {
+            Self::File(config) => config.get_component_name(),
+            Self::Directory(config) => config.get_component_name(),
             Self::Exec(config) => config.get_component_name(),
             #[cfg(feature = "secrets-aws-secrets-manager")]
             Self::AwsSecretsManager(config) => config.get_component_name(),

tests/behavior/config/secret.toml

@@ -6,12 +6,23 @@
   type = "exec"
   command = ["./target/debug/secret-backend-example"]
 
+[secret.file_backend]
+  type = "file"
+  path = "tests/data/secret-backends/file-secrets.json"
+
+[secret.directory_backend]
+  type = "directory"
+  path = "tests/data/secret-backends/directory-secrets"
+  remove_trailing_whitespace = true
+
 [transforms.add_field_from_secret]
   inputs = []
   type = "remap"
   source = '''
   .foobar = "SECRET[test_backend.abc]"
   .foobarbaz = "SECRET[exec_backend.def]"
+  .foobarbazqux = "SECRET[file_backend.ghi]"
+  .foobarbazquxquux = "SECRET[directory_backend.jkl]"
   '''
 
 [[tests]]
@@ -27,4 +38,6 @@
       source = '''
       .foobar == "this_is_a_secret_value"
       .foobarbaz == "def.retrieved"
+      .foobarbazqux == "ghi.retrieved"
+      .foobarbazquxquux == "jkl.retrieved"
       '''

tests/data/secret-backends/directory-secrets/jkl

@@ -0,0 +1 @@
+jkl.retrieved

tests/data/secret-backends/file-secrets.json

@@ -0,0 +1 @@
+{"ghi":"ghi.retrieved"}

website/cue/reference/configuration.cue

@@ -480,6 +480,68 @@ configuration: {
 				"""
 			required: false
 			type: object: options: {
+				file: {
+					required: true
+					description: """
+						Retrieve secrets from a file path.
+
+						The secret must be a JSON text string with key/value pairs. For example:
+						```json
+						{
+							"username": "test",
+							"password": "example-password"
+						}
+						```
+
+						If an error occurs while reading the file, Vector will log the error and exit.
+
+						Secrets are loaded when Vector starts or if Vector receives a `SIGHUP` signal triggering its
+						configuration reload process.
+						"""
+					type: object: options: {
+						path: {
+							description: "The file path to read."
+							required:    true
+							type: string: {
+								examples: ["/path/to/secret.json"]
+							}
+						}
+					}
+				}
+				directory: {
+					required: true
+					description: """
+						Retrieve secrets from file contents in a directory.
+
+						The directory must contain files with names corresponding to secret keys.
+
+						If an error occurs while reading the file, Vector will log the error and exit.
+
+						Secrets are loaded when Vector starts or if Vector receives a `SIGHUP` signal triggering its
+						configuration reload process.
+						"""
+					type: object: options: {
+						path: {
+							description: """
+								The path of the directory with secrets.
+								"""
+							required: true
+							type: string: {
+								examples: [
+									"${CREDENTIALS_DIRECTORY}", // https://systemd.io/CREDENTIALS
+									"/path/to/secrets-directory",
+								]
+							}
+						}
+						remove_trailing_whitespace: {
+							description: """
+								Remove trailing whitespace from file contents.
+								"""
+							required: false
+							type: bool: default: false
+						}
+					}
+				}
 				exec: {
 					required: true
 					description: """