From 9c0229d8bb09d592d64c3f1dc674cf2628a0072a Mon Sep 17 00:00:00 2001 From: Daniel Lea Date: Mon, 30 Oct 2023 10:52:18 +0000 Subject: [PATCH] AB#129760: Working Directory Path (#278) Signed-off-by: Daniel Lea Co-authored-by: Jon Couldridge --- app/HutchAgent/Config/PathOptions.cs | 5 +- .../Config/WorkflowTriggerOptions.cs | 24 +++++++--- .../Constants/ContainerEngineType.cs | 8 ++++ .../Models/DatabaseConnectionDetails.cs | 7 +-- .../Models/Wfexs/WorkflowStageFile.cs | 6 ++- .../Services/WorkflowTriggerService.cs | 12 ++--- .../getting-started/configuration/agent.md | 48 +++++++++++-------- 7 files changed, 71 insertions(+), 39 deletions(-) create mode 100644 app/HutchAgent/Constants/ContainerEngineType.cs diff --git a/app/HutchAgent/Config/PathOptions.cs b/app/HutchAgent/Config/PathOptions.cs index eda2ae04..c3aba92e 100644 --- a/app/HutchAgent/Config/PathOptions.cs +++ b/app/HutchAgent/Config/PathOptions.cs @@ -5,7 +5,9 @@ public class PathOptions /// /// A path for Hutch to use as a working directory. Relative paths start adjacent to the Hutch executable. /// - public string WorkingDirectoryBase { get; set; } = "hutch-workdir"; + public string WorkingDirectoryBase { get; set; } = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + "hutch-workdir"); /// /// A Path to where job working directories should be. @@ -28,4 +30,3 @@ public static class PathOptionsExtensions public static string JobWorkingDirectory(this PathOptions paths, string jobId) => Path.Combine(paths.WorkingDirectoryBase, paths.Jobs, jobId); } - diff --git a/app/HutchAgent/Config/WorkflowTriggerOptions.cs b/app/HutchAgent/Config/WorkflowTriggerOptions.cs index 3250c084..6779c4d1 100644 --- a/app/HutchAgent/Config/WorkflowTriggerOptions.cs +++ b/app/HutchAgent/Config/WorkflowTriggerOptions.cs @@ -1,3 +1,5 @@ +using HutchAgent.Constants; + namespace HutchAgent.Config; public class WorkflowTriggerOptions @@ -5,17 +7,26 @@ public class WorkflowTriggerOptions /// /// Path where Wfexs is installed /// - public string ExecutorPath { get; set; } = string.Empty; + public string ExecutorPath { get; set; } = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + "WfExS-backend"); /// /// Path to the Wfexs virtual environment /// - public string VirtualEnvironmentPath { get; set; } = string.Empty; + public string VirtualEnvironmentPath { get; set; } = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + "WfExS-backend", + ".pyWEenv", + "bin", + "activate"); /// /// Path to the Wfexs local config file /// - public string LocalConfigPath { get; set; } = string.Empty; + public string LocalConfigPath { get; set; } = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + "local_config.yaml"); /// /// Should container images downloaded for workflows be included in the outputs? @@ -43,7 +54,7 @@ public class WorkflowTriggerOptions /// /// public string SkipExecutionUsingOutputFile { get; set; } = string.Empty; - + /// /// Don't ask WfExS for a full provenance output crate (i.e. don't use `--full`). /// `--full` is typically preferred but can be unreliable in some environments, @@ -55,9 +66,8 @@ public class WorkflowTriggerOptions /// The container engine generated stage files should use e.g. `docker` (default), singularity or `podman`. /// Should match the `containerType` configured in the Executor's local config. /// - // TODO enum this - public string ContainerEngine { get; set; } = "docker"; - + public ContainerEngineType ContainerEngine { get; set; } = ContainerEngineType.Docker; + /// /// Keep Hutch attached to Executor processes that it triggers. /// This uses up threads in the pool, but is useful for debugging. diff --git a/app/HutchAgent/Constants/ContainerEngineType.cs b/app/HutchAgent/Constants/ContainerEngineType.cs new file mode 100644 index 00000000..04cb0518 --- /dev/null +++ b/app/HutchAgent/Constants/ContainerEngineType.cs @@ -0,0 +1,8 @@ +namespace HutchAgent.Constants; + +public enum ContainerEngineType +{ + Docker, + Podman, + Singularity +} diff --git a/app/HutchAgent/Models/DatabaseConnectionDetails.cs b/app/HutchAgent/Models/DatabaseConnectionDetails.cs index 35aeb96a..bd79c0bd 100644 --- a/app/HutchAgent/Models/DatabaseConnectionDetails.cs +++ b/app/HutchAgent/Models/DatabaseConnectionDetails.cs @@ -1,4 +1,5 @@ using System.ComponentModel.DataAnnotations; +using HutchAgent.Constants; namespace HutchAgent.Models; @@ -51,14 +52,14 @@ public static class DatabaseConnectionDetailsExtensions /// The object. /// The target container engine. /// - public static string GetContainerHost(this DatabaseConnectionDetails dataAccess, string containerEngine) + public static string GetContainerHost(this DatabaseConnectionDetails dataAccess, ContainerEngineType containerEngine) { return dataAccess.Hostname != "localhost" ? dataAccess.Hostname // if it's not localhost, it's irrelevant; use the configured value : containerEngine switch { - "podman" => "host.containers.internal", - "docker" or "singularity" => "172.17.0.1", + ContainerEngineType.Podman => "host.containers.internal", + ContainerEngineType.Docker or ContainerEngineType.Singularity => "172.17.0.1", _ => dataAccess.Hostname }; } diff --git a/app/HutchAgent/Models/Wfexs/WorkflowStageFile.cs b/app/HutchAgent/Models/Wfexs/WorkflowStageFile.cs index 66abf917..a911a893 100644 --- a/app/HutchAgent/Models/Wfexs/WorkflowStageFile.cs +++ b/app/HutchAgent/Models/Wfexs/WorkflowStageFile.cs @@ -1,3 +1,4 @@ +using HutchAgent.Constants; using YamlDotNet.Serialization; namespace HutchAgent.Models.Wfexs; @@ -18,13 +19,14 @@ public class WorkflowStageFile [YamlMember(Alias = "outputs")] public object Outputs = new(); [YamlMember(Alias = "params")] public object? Params { get; set; } - + [YamlMember(Alias = "environment")] public object EnvironmentVars { get; set; } = new(); } public class Configuration { - [YamlMember(Alias = "container")] public string Container { get; set; } = "docker"; + [YamlMember(Alias = "container")] + public string Container { get; set; } = ContainerEngineType.Docker.ToString().ToLower(); [YamlMember(Alias = "secure")] public bool Secure { get; set; } = false; diff --git a/app/HutchAgent/Services/WorkflowTriggerService.cs b/app/HutchAgent/Services/WorkflowTriggerService.cs index 92b55946..7aedecf5 100644 --- a/app/HutchAgent/Services/WorkflowTriggerService.cs +++ b/app/HutchAgent/Services/WorkflowTriggerService.cs @@ -1,8 +1,7 @@ using System.Diagnostics; +using System.IO.Compression; using System.Text.Json; using System.Text.Json.Nodes; -using System.IO.Abstractions; -using System.IO.Compression; using System.Text.RegularExpressions; using HutchAgent.Config; using HutchAgent.Constants; @@ -279,7 +278,7 @@ private async Task WriteStageFile(WorkflowJob workflowJob, ROCrate roCra throw new NullReferenceException("Cannot find main entity in input crate"); cratePath = Path.Combine(workflowJob.WorkingDirectory.JobCrateRoot(), mainEntity.Id); } - + await InitialiseRepo(cratePath); var workflowCrate = _crateService.InitialiseCrate(cratePath); @@ -295,7 +294,7 @@ private async Task WriteStageFile(WorkflowJob workflowJob, ROCrate roCra Nickname = "HutchAgent" + workflowJob.Id, WorkflowConfig = new() { - Container = _workflowOptions.ContainerEngine // TODO in future validate values + Container = _workflowOptions.ContainerEngine.ToString().ToLower() }, Params = new object() }; @@ -343,7 +342,7 @@ private async Task WriteStageFile(WorkflowJob workflowJob, ROCrate roCra // Set input params stageFile.Params = parameters; - + //Set environment variables var envVars = new Dictionary(); try @@ -353,7 +352,7 @@ private async Task WriteStageFile(WorkflowJob workflowJob, ROCrate roCra // set environment params to match dataAccess details if (dataAccess is not null) { - envVars.Add("DATASOURCE_DB_HOST",dataAccess.GetContainerHost(_workflowOptions.ContainerEngine)); + envVars.Add("DATASOURCE_DB_HOST", dataAccess.GetContainerHost(_workflowOptions.ContainerEngine)); envVars.Add("DATASOURCE_DB_PORT", dataAccess.Port.ToString()); envVars.Add("DATASOURCE_DB_DATABASE", dataAccess.Database); envVars.Add("DATASOURCE_DB_USERNAME", dataAccess.Username); @@ -368,6 +367,7 @@ private async Task WriteStageFile(WorkflowJob workflowJob, ROCrate roCra // it may fail if it intends to try and access the data source, // but that'll get handled and reported in the usual way. } + stageFile.EnvironmentVars = envVars; // Get outputs from workflow crate diff --git a/website/docs/getting-started/configuration/agent.md b/website/docs/getting-started/configuration/agent.md index 871ecc69..25c86edd 100644 --- a/website/docs/getting-started/configuration/agent.md +++ b/website/docs/getting-started/configuration/agent.md @@ -10,6 +10,8 @@ Hutch can be configured using the following source in [the usual .NET way](https - Command line arguments - (.NET User Secrets in development) +Below are the available configurable settings for `HutchAgent`. The variables shown are the defaults, unless specified otherwise. + ## Available values ```json @@ -32,20 +34,24 @@ Hutch can be configured using the following source in [the usual .NET way](https // Local working paths used by Hutch itself "Paths": { - "WorkingDirectoryBase": "hutch-workdir", // Hutch's working directory + "WorkingDirectoryBase": "$HOME/hutch-workdir", // Hutch's working directory "Jobs": "jobs" // Sub-directory for per-job working directories }, // Configuration for Hutch's internal Action Queue (RabbitMQ) + // For actual defaults, see RabbitMQ documentation "Queue": { - "Hostname": "", // Rabbit MQ Host - "Port": 0, // Rabbit MQ Port - "UserName": "", // Rabbit MQ User - "Password": "", // Rabbit MQ Password + "Hostname": "", // RabbitMQ Host + "Port": 0, // RabbitMQ Port + "UserName": "", // RabbitMQ User + "Password": "" // RabbitMQ Password + }, + // Configure the internal queue name, how it checks for jobs, and how many can run concurrently + "JobActionsQueueOptions": { "QueueName": "WorkflowJobActions", - "PollingIntervalSeconds": 5, // How often Hutch checks the internal queue for new Actions - "MaxParallelism": 10 // How many actions from the queue will Hutch run simultaneously + "PollingIntervalSeconds": 5, // How often Hutch checks the queue for new Actions + "MaxParallelism": 10 // How many actions from the queue will Hutch run concurrently }, // MinIO Intermediary Store Defaults @@ -53,10 +59,10 @@ Hutch can be configured using the following source in [the usual .NET way](https // And as a fallback for Submissions/Egress when only partial bucket details are provided. "StoreDefaults": { "Host": "localhost:9000", - "AccessKey": "accesskey", - "SecretKey": "secretkey", - "Secure": false, - "Bucket": "hutch.bucket" + "AccessKey": "", + "SecretKey": "", + "Secure": true, + "Bucket": "" }, "IdentityProvider": { @@ -80,16 +86,16 @@ Hutch can be configured using the following source in [the usual .NET way](https // Configuration for tracking Workflow Execution // Currently WfExS specific "WorkflowExecutor": { - "ExecutorPath": "/WfExS-backend", - "VirtualEnvironmentPath": "/WfExS-backend/.pyWEenv/bin/activate", - "LocalConfigPath": "workflow_examples/local_config.yaml", + "ExecutorPath": "$HOME/WfExS-backend", + "VirtualEnvironmentPath": "$HOME/WfExS-backend/.pyWEenv/bin/activate", + "LocalConfigPath": "$HOME/WfExS-backend/local_config.yaml", "ContainerEngine": "docker", // other valid options are "singularity" and "podman" // The below are more for development / debugging // If a path is provided, Hutch will skip Workflow Execution altogether // and instead use the zip file from this path as if it were the execution output - "SkipExecutionUsingOutputFile": "path/to/exection.crate.zip", + "SkipExecutionUsingOutputFile": "", // e.g. `"path/to/execution.crate.zip"` // Really we always want a full crate, but some wfexs configs // particularly with certain container engines @@ -113,12 +119,17 @@ Hutch can be configured using the following source in [the usual .NET way](https // Configurable details to add to published Results Crates. "CratePublishing": { + // this section defaults to `null` "Publisher": { "Id": "" // Desired Identifier (typically URL) for the Publisher in Results Crates. }, + // this section defaults to `null` "License": { - "Uri": "", // A URI to be used as th License `@id` in Results crate metadata - "Properties": {} // Any valid CreativeWork properties as desirable to be included for the License. + "Uri": "", // A URI to the license e.g. https://spdx.org/licenses/CC-BY-4.0 + "Properties": { + "Identifier": null, // short-form of the license e.g. CC-BY-4.0 + "Name": null // long-form name of the license e.g. Creative Commons Attribution 4.0 International + } } }, @@ -127,8 +138,7 @@ Hutch can be configured using the following source in [the usual .NET way](https // and their continued presence cannot be relied upon from one build to the next! "Flags": { "StandaloneMode": false, // Hutch will skip TRE Controller interactions - "RetainFailures": false, // Hutch will not clean up working directories or database records for jobs that fail. - " + "RetainFailures": false // Hutch will not clean up working directories or database records for jobs that fail. } } ```