AB#129760: Working Directory Path (#278)

Signed-off-by: Daniel Lea <[email protected]>
Co-authored-by: Jon Couldridge <[email protected]>

app/HutchAgent/Config/PathOptions.cs

@@ -5,7 +5,9 @@ public class PathOptions
   /// <summary>
   /// A path for Hutch to use as a working directory. Relative paths start adjacent to the Hutch executable.
   /// </summary>
-  public string WorkingDirectoryBase { get; set; } = "hutch-workdir";
+  public string WorkingDirectoryBase { get; set; } = Path.Combine(
+    Environment.GetFolderPath(Environment.SpecialFolder.UserProfile),
+    "hutch-workdir");
 
   /// <summary>
   /// <para>A Path to where job working directories should be.</para>
@@ -28,4 +30,3 @@ public static class PathOptionsExtensions
   public static string JobWorkingDirectory(this PathOptions paths, string jobId)
     => Path.Combine(paths.WorkingDirectoryBase, paths.Jobs, jobId);
 }
-

app/HutchAgent/Config/WorkflowTriggerOptions.cs

@@ -1,21 +1,32 @@
+using HutchAgent.Constants;
+
 namespace HutchAgent.Config;
 
 public class WorkflowTriggerOptions
 {
   /// <summary>
   /// Path where Wfexs is installed
   /// </summary>
-  public string ExecutorPath { get; set; } = string.Empty;
+  public string ExecutorPath { get; set; } = Path.Combine(
+    Environment.GetFolderPath(Environment.SpecialFolder.UserProfile),
+    "WfExS-backend");
 
   /// <summary>
   /// Path to the Wfexs virtual environment
   /// </summary>
-  public string VirtualEnvironmentPath { get; set; } = string.Empty;
+  public string VirtualEnvironmentPath { get; set; } = Path.Combine(
+    Environment.GetFolderPath(Environment.SpecialFolder.UserProfile),
+    "WfExS-backend",
+    ".pyWEenv",
+    "bin",
+    "activate");
 
   /// <summary>
   /// Path to the Wfexs local config file 
   /// </summary>
-  public string LocalConfigPath { get; set; } = string.Empty;
+  public string LocalConfigPath { get; set; } = Path.Combine(
+    Environment.GetFolderPath(Environment.SpecialFolder.UserProfile),
+    "local_config.yaml");
 
   /// <summary>
   /// Should container images downloaded for workflows be included in the outputs?
@@ -43,7 +54,7 @@ public class WorkflowTriggerOptions
   /// 
   /// </summary>
   public string SkipExecutionUsingOutputFile { get; set; } = string.Empty;
-  
+
   /// <summary>
   /// Don't ask WfExS for a full provenance output crate (i.e. don't use `--full`).
   /// `--full` is typically preferred but can be unreliable in some environments,
@@ -55,9 +66,8 @@ public class WorkflowTriggerOptions
   /// The container engine generated stage files should use e.g. `docker` (default), singularity or `podman`.
   /// Should match the `containerType` configured in the Executor's local config.
   /// </summary>
-  // TODO enum this
-  public string ContainerEngine { get; set; } = "docker";
-
+  public ContainerEngineType ContainerEngine { get; set; } = ContainerEngineType.Docker;
+
   /// <summary>
   /// Keep Hutch attached to Executor processes that it triggers.
   /// This uses up threads in the pool, but is useful for debugging.

app/HutchAgent/Constants/ContainerEngineType.cs

@@ -0,0 +1,8 @@
+namespace HutchAgent.Constants;
+
+public enum ContainerEngineType
+{
+  Docker,
+  Podman,
+  Singularity
+}

app/HutchAgent/Models/DatabaseConnectionDetails.cs

@@ -1,4 +1,5 @@
 using System.ComponentModel.DataAnnotations;
+using HutchAgent.Constants;
 
 namespace HutchAgent.Models;
 
@@ -51,14 +52,14 @@ public static class DatabaseConnectionDetailsExtensions
   /// <param name="dataAccess">The <see cref="DatabaseConnectionDetails" /> object.</param>
   /// <param name="containerEngine">The target container engine.</param>
   /// <returns></returns>
-  public static string GetContainerHost(this DatabaseConnectionDetails dataAccess, string containerEngine)
+  public static string GetContainerHost(this DatabaseConnectionDetails dataAccess, ContainerEngineType containerEngine)
   {
     return dataAccess.Hostname != "localhost"
       ? dataAccess.Hostname // if it's not localhost, it's irrelevant; use the configured value
       : containerEngine switch
       {
-        "podman" => "host.containers.internal",
-        "docker" or "singularity" => "172.17.0.1",
+        ContainerEngineType.Podman => "host.containers.internal",
+        ContainerEngineType.Docker or ContainerEngineType.Singularity => "172.17.0.1",
         _ => dataAccess.Hostname
       };
   }

app/HutchAgent/Models/Wfexs/WorkflowStageFile.cs

@@ -1,3 +1,4 @@
+using HutchAgent.Constants;
 using YamlDotNet.Serialization;
 
 namespace HutchAgent.Models.Wfexs;
@@ -18,13 +19,14 @@ public class WorkflowStageFile
   [YamlMember(Alias = "outputs")] public object Outputs = new();
 
   [YamlMember(Alias = "params")] public object? Params { get; set; }
-  
+
   [YamlMember(Alias = "environment")] public object EnvironmentVars { get; set; } = new();
 }
 
 public class Configuration
 {
-  [YamlMember(Alias = "container")] public string Container { get; set; } = "docker";
+  [YamlMember(Alias = "container")]
+  public string Container { get; set; } = ContainerEngineType.Docker.ToString().ToLower();
 
   [YamlMember(Alias = "secure")] public bool Secure { get; set; } = false;
 

app/HutchAgent/Services/WorkflowTriggerService.cs

@@ -1,8 +1,7 @@
 using System.Diagnostics;
+using System.IO.Compression;
 using System.Text.Json;
 using System.Text.Json.Nodes;
-using System.IO.Abstractions;
-using System.IO.Compression;
 using System.Text.RegularExpressions;
 using HutchAgent.Config;
 using HutchAgent.Constants;
@@ -279,7 +278,7 @@ private async Task<string> WriteStageFile(WorkflowJob workflowJob, ROCrate roCra
                        throw new NullReferenceException("Cannot find main entity in input crate");
       cratePath = Path.Combine(workflowJob.WorkingDirectory.JobCrateRoot(), mainEntity.Id);
     }
-    
+
     await InitialiseRepo(cratePath);
 
     var workflowCrate = _crateService.InitialiseCrate(cratePath);
@@ -295,7 +294,7 @@ private async Task<string> WriteStageFile(WorkflowJob workflowJob, ROCrate roCra
       Nickname = "HutchAgent" + workflowJob.Id,
       WorkflowConfig = new()
       {
-        Container = _workflowOptions.ContainerEngine // TODO in future validate values 
+        Container = _workflowOptions.ContainerEngine.ToString().ToLower()
       },
       Params = new object()
     };
@@ -343,7 +342,7 @@ private async Task<string> WriteStageFile(WorkflowJob workflowJob, ROCrate roCra
 
     // Set input params
     stageFile.Params = parameters;
-    
+
     //Set environment variables
     var envVars = new Dictionary<string, string>();
     try
@@ -353,7 +352,7 @@ private async Task<string> WriteStageFile(WorkflowJob workflowJob, ROCrate roCra
       // set environment params to match dataAccess details
       if (dataAccess is not null)
       {
-        envVars.Add("DATASOURCE_DB_HOST",dataAccess.GetContainerHost(_workflowOptions.ContainerEngine));
+        envVars.Add("DATASOURCE_DB_HOST", dataAccess.GetContainerHost(_workflowOptions.ContainerEngine));
         envVars.Add("DATASOURCE_DB_PORT", dataAccess.Port.ToString());
         envVars.Add("DATASOURCE_DB_DATABASE", dataAccess.Database);
         envVars.Add("DATASOURCE_DB_USERNAME", dataAccess.Username);
@@ -368,6 +367,7 @@ private async Task<string> WriteStageFile(WorkflowJob workflowJob, ROCrate roCra
       // it may fail if it intends to try and access the data source,
       // but that'll get handled and reported in the usual way.
     }
+
     stageFile.EnvironmentVars = envVars;
 
     // Get outputs from workflow crate

website/docs/getting-started/configuration/agent.md

@@ -10,6 +10,8 @@ Hutch can be configured using the following source in [the usual .NET way](https
 - Command line arguments
 - (.NET User Secrets in development)
 
+Below are the available configurable settings for `HutchAgent`. The variables shown are the defaults, unless specified otherwise.
+
 ## Available values
 
 ```json
@@ -32,31 +34,35 @@ Hutch can be configured using the following source in [the usual .NET way](https
 
   // Local working paths used by Hutch itself
   "Paths": {
-    "WorkingDirectoryBase": "hutch-workdir", // Hutch's working directory
+    "WorkingDirectoryBase": "$HOME/hutch-workdir", // Hutch's working directory
     "Jobs": "jobs" // Sub-directory for per-job working directories
   },
 
   // Configuration for Hutch's internal Action Queue (RabbitMQ)
+  // For actual defaults, see RabbitMQ documentation
   "Queue": {
-    "Hostname": "", // Rabbit MQ Host
-    "Port": 0, // Rabbit MQ Port
-    "UserName": "", // Rabbit MQ User
-    "Password": "", // Rabbit MQ Password
+    "Hostname": "", // RabbitMQ Host
+    "Port": 0, // RabbitMQ Port
+    "UserName": "", // RabbitMQ User
+    "Password": "" // RabbitMQ Password
+  },
 
+  // Configure the internal queue name, how it checks for jobs, and how many can run concurrently
+  "JobActionsQueueOptions": {
     "QueueName": "WorkflowJobActions",
-    "PollingIntervalSeconds": 5, // How often Hutch checks the internal queue for new Actions
-    "MaxParallelism": 10 // How many actions from the queue will Hutch run simultaneously
+    "PollingIntervalSeconds": 5, // How often Hutch checks the queue for new Actions
+    "MaxParallelism": 10 // How many actions from the queue will Hutch run concurrently
   },
 
   // MinIO Intermediary Store Defaults
   // These are are used for Egress in Standalone Mode
   // And as a fallback for Submissions/Egress when only partial bucket details are provided.
   "StoreDefaults": {
     "Host": "localhost:9000",
-    "AccessKey": "accesskey",
-    "SecretKey": "secretkey",
-    "Secure": false,
-    "Bucket": "hutch.bucket"
+    "AccessKey": "",
+    "SecretKey": "",
+    "Secure": true,
+    "Bucket": ""
   },
 
   "IdentityProvider": {
@@ -80,16 +86,16 @@ Hutch can be configured using the following source in [the usual .NET way](https
   // Configuration for tracking Workflow Execution
   // Currently WfExS specific
   "WorkflowExecutor": {
-    "ExecutorPath": "/WfExS-backend",
-    "VirtualEnvironmentPath": "/WfExS-backend/.pyWEenv/bin/activate",
-    "LocalConfigPath": "workflow_examples/local_config.yaml",
+    "ExecutorPath": "$HOME/WfExS-backend",
+    "VirtualEnvironmentPath": "$HOME/WfExS-backend/.pyWEenv/bin/activate",
+    "LocalConfigPath": "$HOME/WfExS-backend/local_config.yaml",
     "ContainerEngine": "docker", // other valid options are "singularity" and "podman"
 
     // The below are more for development / debugging
 
     // If a path is provided, Hutch will skip Workflow Execution altogether
     // and instead use the zip file from this path as if it were the execution output
-    "SkipExecutionUsingOutputFile": "path/to/exection.crate.zip",
+    "SkipExecutionUsingOutputFile": "", // e.g. `"path/to/execution.crate.zip"`
 
     // Really we always want a full crate, but some wfexs configs
     // particularly with certain container engines
@@ -113,12 +119,17 @@ Hutch can be configured using the following source in [the usual .NET way](https
 
   // Configurable details to add to published Results Crates.
   "CratePublishing": {
+    // this section defaults to `null`
     "Publisher": {
       "Id": "" // Desired Identifier (typically URL) for the Publisher in Results Crates.
     },
+    // this section defaults to `null`
     "License": {
-      "Uri": "", // A URI to be used as th License `@id` in Results crate metadata
-      "Properties": {} // Any valid CreativeWork properties as desirable to be included for the License.
+      "Uri": "", // A URI to the license e.g. https://spdx.org/licenses/CC-BY-4.0
+      "Properties": {
+        "Identifier": null, // short-form of the license e.g. CC-BY-4.0
+        "Name": null // long-form name of the license e.g. Creative Commons Attribution 4.0 International
+      }
     }
   },
 
@@ -127,8 +138,7 @@ Hutch can be configured using the following source in [the usual .NET way](https
   // and their continued presence cannot be relied upon from one build to the next!
   "Flags": {
     "StandaloneMode": false, // Hutch will skip TRE Controller interactions
-    "RetainFailures": false, // Hutch will not clean up working directories or database records for jobs that fail.
-    "
+    "RetainFailures": false // Hutch will not clean up working directories or database records for jobs that fail.
   }
 }
 ```