Merge pull request #13 from davehorner/multipart-stdin-example-enviro…

…nment-variable-hooks

multipart-stdin example, hook environment vars, LICENSE, and README

Cargo.toml

@@ -29,6 +29,9 @@ sha2 = "0.10.2"
 tokio = { version = "1.24.2", features = ["rt", "process", "net", "io-util", "macros", "signal", "time"] }
 tokio-openssl = "0.6.2"
 tokio-stream = { version = "0.1.7", features = ["io-util"] }
+indexmap = { version = "2.0.2", features = ["serde"] }
 
 [dev-dependencies]
 assert2 = "0.3.6"
+multer = { version="2.1.0", features = ["tokio-io"] }
+tokio = { version = "1.24.2", features = ["rt", "process", "net", "io-util", "macros", "signal", "time", "io-std","rt-multi-thread"] }

LICENSE.md

@@ -1,6 +1,7 @@
 Copyright 2020, Maarten de Vries <[email protected]>
 
 Copyright 2020, Maarten de Vries <[email protected]>
+Copyright 2023, David Horner
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:

README.md

@@ -69,3 +69,36 @@ The crate has one optional feature: `static-openssl`.
 When the feature is enabled, `openssl` is linked statically against a locally compiled OpenSSL.
 This can be used to create a binary with a minimal set of runtime dependencies,
 and it can make compilation easier on systems with no recent version of OpenSSL readily available.
+
+On windows, this has been reported to work.
+```
+choco install openssl
+set OPENSSL_DIR=C:\Program Files\OpenSSL-Win64
+```
+
+## Examples
+The `multipart-stdin` example shows how to process `multipart/form-data` from stdin and how to pass additional environment variables to your hooks from the config file.
+
+Build the example
+```sh
+ cargo build --example multipart-stdin --features static-openssl
+```
+
+Add the hook (windows example)
+```
+  - url: "/multipart-stdin"
+    commands:
+      - cmd: ["cmd.exe","/c","target\\debug\\examples\\multipart-stdin.exe"]
+        stdin: request-body
+    environment: ["OUTPUT_FOLDER=uploads","COUNT_SUFFIX","APPLY_TIMESTAMP"]
+```
+
+Run the server:
+```sh
+cargo run --features static-openssl -- --config example-config.yaml
+```
+
+You can test the endpoint using `curl` with the `-F` option:
+```sh
+curl -X POST -F "key1=value1" -F "key2=value2" -F "[email protected]"   http://localhost:8091/multipart-stdin
+```

example-config.yaml

@@ -11,9 +11,9 @@ log-level: info
 
 # The TLS section is optional.
 # If it is present, the server will only accept HTTPS requests.
-tls:
-  private-key: /etc/letsencrypt/live/example.com/privkey.pem
-  certificate-chain: /etc/letsencrypt/live/example.com/fullchain.pem
+#tls:
+#  private-key: /etc/letsencrypt/live/example.com/privkey.pem
+#  certificate-chain: /etc/letsencrypt/live/example.com/fullchain.pem
 
 # The hooks.
 hooks:
@@ -23,6 +23,27 @@ hooks:
     commands:
       - cmd: ["env"]
 
+  - url: "/print-env-win32"
+    commands:
+      - cmd: ["cmd.exe","/c","set"]
+
+  - url: "/print-cwd"
+    commands:
+      - cmd: ["pwd"]
+
+  - url: "/print-cwd-win32"
+    commands:
+      - cmd: ["cmd.exe","/c","cd"]
+
+  - url: "/multipart-stdin"
+    commands:
+      - cmd: ["target/debug/examples/multipart-stdin"]
+        stdin: request-body
+    environment: 
+      OUTPUT_FOLDER: uploads
+      COUNT_SUFFIX: 1
+      APPLY_TIMESTAMP: 1
+
   - url: "/make-release-tarball"
     commands:
       - cmd: ["make-release-tarball"]

examples/multipart-stdin.rs

@@ -0,0 +1,110 @@
+use tokio::io;
+use multer::Multipart;
+use std::fs::File;
+use std::path::{Path, PathBuf};
+use std::io::Write;
+use std::fs;
+use chrono::Local;
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // Read the CONTENT_TYPE environment variable to get the boundary
+    let content_type = match std::env::var("CONTENT_TYPE") {
+        Ok(content_type) => content_type,
+        Err(_) => {
+            eprintln!("Error: CONTENT_TYPE environment variable not found.");
+            return Ok(());
+        }
+    };
+
+    // Parse the boundary from the content_type, exit early if we don't have it.
+    let boundary = match content_type.split(';').find(|s| s.trim().starts_with("boundary=")) {
+        Some(boundary) => boundary.trim().trim_start_matches("boundary=").to_string(),
+        None => {
+            eprintln!("Error: Boundary not found in CONTENT_TYPE.");
+            return Ok(());
+        }
+    };
+
+    // Read the OUTPUT_FOLDER environment variable to store files somewhere
+    let output_folder = std::env::var("OUTPUT_FOLDER").unwrap_or_default();
+	if !output_folder.is_empty() {
+	    println!("writing to: {}",output_folder);
+		if !Path::new(&output_folder).is_dir() {
+        	if let Err(err) = fs::create_dir_all(&output_folder) {
+            	eprintln!("Error creating directory: {}", err);
+        	} else {
+            	println!("Directory '{}' created successfully.", output_folder);
+        	}
+	    }
+    } else {
+        println!("No output folder specified.");
+    }
+
+    let mut multipart = Multipart::with_reader(io::stdin(), &boundary);
+    while let Some(mut field) = multipart.next_field().await? {
+        let field_name = field.name().unwrap_or_default().to_string();
+        let has_filename= field.file_name().map(|s| s.to_string());
+
+        // Check if the field has a file name
+        if let Some(file_name) = has_filename {
+            println!("Writing a file: {}", &file_name);
+
+			let apply_timestamp = std::env::var("APPLY_TIMESTAMP").is_ok();
+			let timestamp = if apply_timestamp {
+                format!("_{}", Local::now().format("%Y%m%d%H%M"))
+            } else {
+                "".to_string()
+            };
+            let file_extension = Path::new(&file_name)
+                .extension()
+                .map(|ext| ext.to_str().unwrap_or(""))
+                .unwrap_or("");
+			let file_name_noext = extract_filename_without_ext(&file_name);
+            let mut unique_name = format!("{}{}.{}", file_name_noext, timestamp, file_extension);
+            let mut unique_file_name = Path::new(&output_folder).join(unique_name);
+
+			// Loop until a unique filename is found
+			let count_suffix = std::env::var("COUNT_SUFFIX").is_ok();
+			if count_suffix {
+				let mut counter = 0;
+				while unique_file_name.exists() {
+                	unique_name = format!("{}{}.{}.{}", file_name_noext, timestamp, counter, file_extension);
+                	unique_file_name = PathBuf::from(&output_folder).join(&unique_name);
+                	counter += 1;
+            	}
+			}
+
+            if let Ok(mut file) = File::create(&unique_file_name) {
+                while let Some(chunk) = field.chunk().await? {
+                    if let Err(e) = file.write_all(&chunk) {
+                        eprintln!("Error writing to file: {}", e);
+                        break;
+                    }
+                }
+                println!("File '{}' uploaded and saved as '{:?}'", &file_name, &unique_file_name);
+            } else {
+                eprintln!("Error creating file: {}", &file_name);
+            }
+        } else {
+            while let Some(chunk) = field.chunk().await? {
+                println!("Field '{}' = {}", field_name, String::from_utf8_lossy(&chunk));
+            }
+        }
+    }
+
+    Ok(())
+}
+
+fn extract_filename_without_ext(file_name: &str) -> String {
+    let file_path = Path::new(file_name);
+
+    if let Some(file_stem) = file_path.file_stem() {
+        if let Some(file_stem_str) = file_stem.to_str() {
+            return file_stem_str.to_string();
+        }
+    }
+
+    String::new()
+}
+

src/config.rs

@@ -1,6 +1,7 @@
 use serde::Deserialize;
 use std::path::{Path, PathBuf};
 use std::net::IpAddr;
+use indexmap::IndexMap;
 
 use crate::logging::LogLevel;
 use crate::types::QueueType;
@@ -50,6 +51,10 @@ pub struct Hook {
 	/// The commands to execute when the hook is triggered.
 	pub commands: Vec<Command>,
 
+	/// The environment variables to set when the hook is triggered.
+	#[serde(default)]
+	pub environment: IndexMap<String, String>,
+
 	/// The maxmimum number of concurrent jobs for the hook.
 	///
 	/// Can be a number or `unlimited`.

src/main.rs

@@ -306,6 +306,11 @@ async fn run_command(cmd: &config::Command, hook: &Hook, request: &Request, body
 		set_request_environment(&mut command, request, None, remote_addr);
 	}
 
+	// Set hook environment variables
+	for (name, value) in &hook.environment {
+		command.env(name, value);
+	}
+
 	let mut subprocess = command.spawn().map_err(|e| log::error!("{}: failed to run command {:?}: {}", hook.url, cmd.cmd(), e))?;
 
 	if let Some(mut stdin) = subprocess.stdin.take() {