From 32e0a13f5f129aeb90415f464c6aa81635a25793 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Tue, 9 Jan 2018 08:36:39 -0800 Subject: [PATCH 01/15] Remove dead code --- .gitmodules | 3 - .../swagger/ga4gh-tool-discovery.yaml | 313 ------------------ task-execution-schemas | 1 - 3 files changed, 317 deletions(-) delete mode 100644 src/main/resources/swagger/ga4gh-tool-discovery.yaml delete mode 160000 task-execution-schemas diff --git a/.gitmodules b/.gitmodules index a312dad3..e69de29b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "task-execution-schemas"] - path = task-execution-schemas - url = https://github.com/ga4gh/task-execution-schemas.git diff --git a/src/main/resources/swagger/ga4gh-tool-discovery.yaml b/src/main/resources/swagger/ga4gh-tool-discovery.yaml deleted file mode 100644 index 75e0906f..00000000 --- a/src/main/resources/swagger/ga4gh-tool-discovery.yaml +++ /dev/null @@ -1,313 +0,0 @@ -swagger: '2.0' -info: - title: GA4GH Workflow Execution API - description: 'Proposed API for GA4GH workflow execution. A workflow is defined with a given format (currently CWL or WDL) and can encompass a single tool execution or a complex DAG of individual tool calls. The parameterization of the workflow called will follow the schema proposed in this group as a common format used for WDL and CWL execution' - version: 0.1.0 -produces: - - application/json - - text/plain -basePath: /api/v1 -tags: - - name: GA4GH - description: A set of resources proposed as a common standard for tool repositories -paths: - /tools/{registry-id}: - get: - summary: List one specific tool, acts as an anchor for self references - description: This endpoint returns one specific tool (which has ToolVersions nested inside it) - tags: - - GA4GH - parameters: - - name: registry-id - in: path - required: true - type: string - description: A unique identifier of the tool for this particular tool registry, for example `123456` - responses: - '200': - description: A tool. - schema: - $ref: '#/definitions/Tool' - - /tools/{registry-id}/version/{version-id}: - get: - summary: List one specific tool version, acts as an anchor for self references - description: This endpoint returns one specific tool version - tags: - - GA4GH - parameters: - - name: registry-id - in: path - required: true - type: string - description: A unique identifier of the tool for this particular tool registry, for example `123456` - - name: version-id - in: path - required: true - type: string - description: An identifier of the tool version for this particular tool registry, for example `v1` - responses: - '200': - description: A tool version. - schema: - $ref: '#/definitions/ToolVersion' - - /tools: - get: - summary: List all tools - description: > - This endpoint returns all tools available or a filtered subset using - metadata query parameters. - tags: - - GA4GH - parameters: - - name: registry-id - type: string - in: query - description: A unique identifier of the tool for this particular tool registry, for example `123456` - - name: registry - in: query - type: string - description: The image registry that contains the image. - - name: organization - in: query - type: string - description: The organization in the registry that published the image. - - name: name - in: query - type: string - description: The name of the image. - - name: toolname - in: query - type: string - description: The name of the tool. - - name: description - in: query - type: string - description: The description of the tool. - - name: author - in: query - type: string - description: 'The author of the tool (TODO a thought occurs, are we assuming that the author of the CWL and the image are the same?).' - responses: - '200': - description: An array of methods that match the filter. - schema: - type: array - items: - $ref: '#/definitions/Tool' - /tools/{registry-id}/version/{version-id}/descriptor: - get: - summary: Get the tool descriptor (CWL/WDL) for the specified tool. - description: Returns the CWL or WDL descriptor for the specified tool. - tags: - - GA4GH - parameters: - - name: format - in: query - description: The output type of the descriptor. If not specified it is up to the underlying implementation to determine which output format to return. - type: string - enum: - - CWL - - WDL - - name: registry-id - in: path - description: A unique identifier of the tool for this particular tool registry, for example `123456` - required: true - type: string - - name: version-id - in: path - required: true - type: string - description: An identifier of the tool version for this particular tool registry, for example `v1` - responses: - '200': - description: The tool descriptor. - schema: - $ref: '#/definitions/ToolDescriptor' - '404': - description: The tool can not be output in the specified format. - schema: - $ref: '#/definitions/Error' - /tools/{registry-id}/version/{version-id}/dockerfile: - get: - summary: Get the dockerfile for the specified image. - description: Returns the dockerfile for the specified image. - tags: - - GA4GH - parameters: - - name: registry-id - in: path - description: A unique identifier of the tool for this particular tool registry, for example `123456` - required: true - type: string - - name: version-id - in: path - required: true - type: string - description: An identifier of the tool version for this particular tool registry, for example `v1` - responses: - '200': - description: The tool payload. - schema: - $ref: '#/definitions/ToolDockerfile' - '404': - description: The tool payload is not present in the service. - schema: - $ref: '#/definitions/Error' - - /tools/metadata: - get: - summary: Return some metadata that is useful for describing this registry - description: Return some metadata that is useful for describing this registry - tags: - - GA4GH - responses: - '200': - description: A Metadata object describing this service. - schema: - $ref: '#/definitions/Metadata' - -definitions: - ToolType: - # thought. it would be nicer if ToolTypes are consistent across registries. How would we accomplish this? Operate a MIME-type like list? - description: Describes a type of tool allowing us to categorize workflows, the language of the workflow, tools, and maybe even other entities separately - properties: - id: - type: string - description: The unique identifier for the type - name: - type: string - description: A short friendly name for the type - description: - type: string - description: A longer explanation of what this type is and what it can accomplish - Tool: - description: A tool (or described tool) describes one pairing of a tool as described in a descriptor file (which potentially describes multiple tools) and a Docker image. - required: - - global-id - - registry-id - - registry - - organization - - name - - author - - meta-version - - tooltype - properties: - global-id: - type: string - description: The unique identifier for the image. (Proposed - This id should be globally unique across systems and should also identify the system that it comes from for example This id should be globally unique across systems, should also identify the system that it comes from, and be a URL that resolves for example `http://agora.broadinstitute.org/tools/123456`) - registry-id: - type: string - description: A unique identifier of the tool for this particular tool registry, for example `123456` or `123456_v1` - registry: - type: string - description: The registry that contains the image. - organization: - type: string - description: The organization that published the image. - name: - type: string - description: The name of the image. - toolname: - type: string - description: The name of the tool. - tooltype: - $ref: '#/definitions/ToolType' - description: - type: string - description: The description of the tool. - author: - type: string - description: Contact information for the author of this tool entry in the registry. (More complex authorship information is handled by the descriptor) - meta-version: - type: string - description: 'The version of this tool in the registry. Iterates when fields like the description, author, etc. are updated.' - contains: - description: An array of IDs for the applications that are stored inside this tool (for example `https://bio.tools/tool/mytum.de/SNAP2/1`) - type: array - items: - type: string - versions: - type: array - items: - $ref: '#/definitions/ToolVersion' - ToolVersion: - description: A tool version describes a particular iteration of a tool as described by a reference to a specific image and dockerfile. - required: - - global-id - - descriptor - - registry-id - - image - - meta-version - properties: - name: - type: string - description: The name of the version. - global-id: - type: string - description: The unique identifier for this version of a tool. (Proposed - This id should be globally unique across systems and should also identify the system that it comes from for example This id should be globally unique across systems, should also identify the system that it comes from, and be a URL that resolves for example `http://agora.broadinstitute.org/tools/123456/v1` This can be the same as the registry-id depending on the structure of your registry) - registry-id: - type: string - description: An identifier of the version of this tool for this particular tool registry, for example `v1` - image: - type: string - description: The docker path to the image (and version) for this tool. (e.g. quay.io/seqware/seqware_full/1.1) - descriptor: - $ref: '#/definitions/ToolDescriptor' - dockerfile: - $ref: '#/definitions/ToolDockerfile' - meta-version: - type: string - description: 'The version of this tool version in the registry. Iterates when fields like the description, author, etc. are updated.' - ToolDescriptor: - description: A tool descriptor is a metadata document that describes one or more tools. - required: - - descriptor - properties: - descriptor: - type: string - description: The descriptor that represents this version of the tool. (CWL or WDL) - url: - type: string - description: 'Optional url to the tool descriptor used to build this image, should include version information, and can include a git hash (e.g. https://raw.githubusercontent.com/ICGC-TCGA-PanCancer/pcawg_delly_workflow/ea2a5db69bd20a42976838790bc29294df3af02b/delly_docker/Delly.cwl )' - ToolDockerfile: - description: A tool dockerfile is a document that describes how to build a particular Docker image. - required: - - dockerfile - properties: - dockerfile: - type: string - description: The dockerfile content for this tool. - url: - type: string - description: 'Optional url to the dockerfile used to build this image, should include version information, and can include a git hash (e.g. https://raw.githubusercontent.com/ICGC-TCGA-PanCancer/pcawg_delly_workflow/c83478829802b4d36374870843821abe1b625a71/delly_docker/Dockerfile )' - Metadata: - description: Describes this registry to better allow for mirroring and indexing. - required: - - version - properties: - version: - type: string - description: The version of this registry - country: - type: string - description: A country code for the registry (ISO 3166-1 alpha-3) - friendly-name: - type: string - description: A friendly name that can be used in addition to the hostname to describe a registry - - Error: - required: - - code - properties: - code: - type: integer - format: int32 - default: 500 - message: - type: string - default: Internal Server Error -externalDocs: - description: Description of GA4GH Tool Registry (Exchange) Schema - url: 'https://github.com/ga4gh/tool-registry-schemas' diff --git a/task-execution-schemas b/task-execution-schemas deleted file mode 160000 index ba01c73a..00000000 --- a/task-execution-schemas +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ba01c73ab9fd54624f83a67b403f85f694f989fa From 9a2771ea8a0a06d23731c1c46a1f1a74bd9e34c5 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Tue, 9 Jan 2018 16:00:16 -0800 Subject: [PATCH 02/15] Remove reference to TES in CWLfile and proto --- CWLFile | 2 +- proto/workflow_execution.proto | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CWLFile b/CWLFile index 65ef51ce..ff77e9a7 100644 --- a/CWLFile +++ b/CWLFile @@ -2,7 +2,7 @@ includes: - class: Directory location: proto - class: Directory - location: task-execution-schemas/proto + location: proto proto: class: File location: proto/workflow_execution.proto diff --git a/proto/workflow_execution.proto b/proto/workflow_execution.proto index 50500b48..6f6e8cd8 100644 --- a/proto/workflow_execution.proto +++ b/proto/workflow_execution.proto @@ -6,7 +6,6 @@ package ga4gh_wes_; // Import HTTP RESTful annotations import "google/api/annotations.proto"; import "google/protobuf/struct.proto"; -import "task_execution.proto"; //Enum for states enum state { From 0f28f4993d8d41ef5291e1b32dabfc7af515c510 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Tue, 9 Jan 2018 16:22:36 -0800 Subject: [PATCH 03/15] Rename to service --- CWLFile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CWLFile b/CWLFile index ff77e9a7..e869df0d 100644 --- a/CWLFile +++ b/CWLFile @@ -5,5 +5,5 @@ includes: location: proto proto: class: File - location: proto/workflow_execution.proto + location: proto/workflow_execution_service.proto cwl:tool: protoc.cwl \ No newline at end of file From 0a7a1c9960caabe2873b5e91ff8734d900997662 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Tue, 9 Jan 2018 16:25:03 -0800 Subject: [PATCH 04/15] Move service to top, comment formatting --- proto/workflow_execution.proto | 189 -------------------- proto/workflow_execution_service.proto | 236 +++++++++++++++++++++++++ 2 files changed, 236 insertions(+), 189 deletions(-) delete mode 100644 proto/workflow_execution.proto create mode 100644 proto/workflow_execution_service.proto diff --git a/proto/workflow_execution.proto b/proto/workflow_execution.proto deleted file mode 100644 index 6f6e8cd8..00000000 --- a/proto/workflow_execution.proto +++ /dev/null @@ -1,189 +0,0 @@ - -syntax = "proto3"; - -package ga4gh_wes_; - -// Import HTTP RESTful annotations -import "google/api/annotations.proto"; -import "google/protobuf/struct.proto"; - -//Enum for states -enum state { - Unknown = 0; - Queued = 1; - Running = 2; - Paused = 3; - Complete = 4; - Error = 5; - SystemError = 6; - Canceled = 7; - Initializing = 8; -} - -//Log and other info -message log { - //The task or workflow name - string name = 1; - //The command line that was run - repeated string cmd = 2; - //When the command was executed - string startTime = 3; - //When the command completed - string endTime = 4; - //Sample of stdout (not guaranteed to be entire log) - string stdout = 5; - //Sample of stderr (not guaranteed to be entire log) - string stderr = 6; - //Exit code of the program - int32 exitCode = 7; -} - -//Return envelope for workflow listing -message workflow_list_response { - repeated workflow_desc workflows = 1; - string next_page_token = 2; -} - -//Request listing of jobs tracked by server -message workflow_list_request { - //OPTIONAL - //Number of workflows to return at once. Defaults to 256, and max is 2048. - uint32 page_size = 1; - //OPTIONAL - //Token to use to indicate where to start getting results. If unspecified, returns the first page of results. - string page_token = 2; - //OPTIONAL - //For each key, if the key's value is empty string then match workflows that are tagged with this key regardless of value - string key_value_search = 3; -} - -//Small description of workflows, returned by server during listing -message workflow_desc { - //REQUIRED - string workflow_id = 1; - //REQUIRED - state state = 2; -} - -//workflow request object -message workflow_request { - //OPTIONAL - //the workflow CWL or WDL document, must provide either this or workflow_url - string workflow_descriptor = 1; - //REQUIRED - //the workflow parameterization document (typically a JSON file), includes all parameterizations for the workflow including input and output file locations - google.protobuf.Struct workflow_params = 2; - //REQUIRED - //the workflow descriptor type, must be "CWL" or "WDL" currently (or another alternative supported by this WES instance) - string workflow_type = 3; - //REQUIRED - //the workflow descriptor type version, must be one supported by this WES instance - string workflow_type_version = 4; - //OPTIONAL - //a key-value map of arbitrary metadata outside the scope of the workflow_params but useful to track with this workflow request - map key_values = 5; - //OPTIONAL - //the workflow CWL or WDL document, must provide either this or workflow_descriptor - string workflow_url = 6; -} - -//Blank request message for service request -message service_info_request {} - -//available workflow types supported by this WES -message workflow_type_version { - //an array of one or more version strings - repeated string workflow_type_version = 1; -} - - -//. -message service_info { - //A map with keys as the workflow format type name (currently only CWL and WDL are used although a service may support others) and value is a workflow_type_version object which simply contains an array of one or more version strings - map workflow_type_versions = 1; - //The version(s) of the WES schema supported by this service - repeated string supported_wes_versions = 2; - //The filesystem protocols supported by this service, currently these may include common protocols such as 'http', 'https', 'sftp', 's3', 'gs', 'file', 'synapse', or others as supported by this service. - repeated string supported_filesystem_protocols = 3; - //The engine(s) used by this WES service, key is engine name e.g. Cromwell and value is version - map engine_versions = 4; - //The system statistics, key is the statistic, value is the count of workflows in that state. See the State enum for the possible keys. - map system_state_counts = 5; - //a key-value map of arbitrary, extended metadata outside the scope of the above but useful to report back - map key_values = 6; -} - -message workflow_run_id { - //workflow ID - string workflow_id = 1; -} - -message workflow_status { - //workflow ID - string workflow_id = 1; - //state - state state = 2; -} - -message workflow_log { - //workflow ID - string workflow_id = 1; - //the original request object - workflow_request request = 2; - //state - state state = 3; - //the logs, and other key info like timing and exit code, for the overall run of this workflow - log workflow_log = 4; - //the logs, and other key info like timing and exit code, for each step in the workflow - repeated log task_logs = 5; - //the outputs - google.protobuf.Struct outputs = 6; -} - -//Web service to get, create, list and delete Workflows -service WorkflowExecutionService { - - //Get information about Workflow Execution Service. May include information related (but not limited to) the workflow descriptor formats, versions supported, the WES API versions supported, and information about general the service availability. - rpc GetServiceInfo(service_info_request) returns (service_info) { - option (google.api.http) = { - get: "/ga4gh/wes/v1/service-info" - }; - } - - //Run a workflow, this endpoint will allow you to create a new workflow request and retrieve its tracking ID to monitor its progress. An important assumption in this endpoint is that the workflow_params JSON will include parameterizations along with input and output files. The latter two may be on S3, Google object storage, local filesystems, etc. This specification makes no distinction. However, it is assumed that the submitter is using URLs that this system both understands and can access. For Amazon S3, this could be accomplished by given the credentials associated with a WES service access to a particular bucket. The details are important for a production system and user on-boarding but outside the scope of this spec. - rpc RunWorkflow(workflow_request) returns (workflow_run_id) { - option (google.api.http) = { - post: "/ga4gh/wes/v1/workflows" - body: "*" - }; - } - - //Get quick status info about a running workflow - rpc GetWorkflowStatus(workflow_run_id) returns (workflow_status) { - option (google.api.http) = { - get: "/ga4gh/wes/v1/workflows/{workflow_id}/status" - }; - } - - //Get detailed info about a running workflow - rpc GetWorkflowLog(workflow_run_id) returns (workflow_log) { - option (google.api.http) = { - get: "/ga4gh/wes/v1/workflows/{workflow_id}" - }; - } - - //Cancel a running workflow - rpc CancelJob(workflow_run_id) returns (workflow_run_id) { - option (google.api.http) = { - delete: "/ga4gh/wes/v1/workflows/{workflow_id}" - }; - } - - //List the workflows, this endpoint will list the workflows in order of oldest to newest. There is no guarantee of live updates as the user traverses the pages, the behavior should be decided (and documented) by each implementation. - rpc ListWorkflows(workflow_list_request) returns (workflow_list_response) { - option (google.api.http) = { - get: "/ga4gh/wes/v1/workflows" - }; - } - -} diff --git a/proto/workflow_execution_service.proto b/proto/workflow_execution_service.proto new file mode 100644 index 00000000..b244664a --- /dev/null +++ b/proto/workflow_execution_service.proto @@ -0,0 +1,236 @@ + +syntax = "proto3"; + +package ga4gh_wes_; + +// Import HTTP RESTful annotations +import "google/api/annotations.proto"; +import "google/protobuf/struct.proto"; + + +// The Workflow Execution Service presents an interoperable layer for executing workflows +// using an HTTP interface. By passing WES messages to a service connected to some execution +// engine one can carry out a computational task. +service WorkflowExecutionService { + + // Get information about Workflow Execution Service. May include information related (but + // not limited to) the workflow descriptor formats, versions supported, the WES API versions supported, and information about general the service availability. + rpc GetServiceInfo(service_info_request) returns (service_info) { + option (google.api.http) = { + get: "/ga4gh/wes/v1/service-info" + }; + } + + // Run a workflow, this endpoint will allow you to create a new workflow request and + // retrieve its tracking ID to monitor its progress. An important assumption in this + // endpoint is that the workflow_params JSON will include parameterizations along with + // input and output files. The latter two may be on S3, Google object storage, local filesystems, + // etc. This specification makes no distinction. However, it is assumed that the submitter + // is using URLs that this system both understands and can access. For Amazon S3, this could + // be accomplished by given the credentials associated with a WES service access to a + // particular bucket. The details are important for a production system and user on-boarding + // but outside the scope of this spec. + rpc RunWorkflow(workflow_request) returns (workflow_run_id) { + option (google.api.http) = { + post: "/ga4gh/wes/v1/workflows" + body: "*" + }; + } + + // Get quick status info about a running workflow. + rpc GetWorkflowStatus(workflow_run_id) returns (workflow_status) { + option (google.api.http) = { + get: "/ga4gh/wes/v1/workflows/{workflow_id}/status" + }; + } + + // Get detailed info about a running workflow. + rpc GetWorkflowLog(workflow_run_id) returns (workflow_log) { + option (google.api.http) = { + get: "/ga4gh/wes/v1/workflows/{workflow_id}" + }; + } + + // Cancel a running workflow. + rpc CancelJob(workflow_run_id) returns (workflow_run_id) { + option (google.api.http) = { + delete: "/ga4gh/wes/v1/workflows/{workflow_id}" + }; + } + + // List the workflows, this endpoint will list the workflows in order of oldest to newest. + // There is no guarantee of live updates as the user traverses the pages, the behavior should be + // decided (and documented) by each implementation. + // To monitor a given execution, use GetWorkflowStatus or GetWorkflowLog. + rpc ListWorkflows(workflow_list_request) returns (workflow_list_response) { + option (google.api.http) = { + get: "/ga4gh/wes/v1/workflows" + }; + } + +} + +// To execute a workflow, send a workflow request including all the details needed to begin downloading +// and executing a given workflow. +message workflow_request { + // OPTIONAL + // The workflow CWL or WDL document, must provide either this or workflow_url + string workflow_descriptor = 1; + + // REQUIRED + // The workflow parameterization document (typically a JSON file), includes all parameterizations for the workflow including input and output file locations + google.protobuf.Struct workflow_params = 2; + + // REQUIRED + // The workflow descriptor type, must be "CWL" or "WDL" currently (or another alternative supported by this WES instance) + string workflow_type = 3; + + // REQUIRED + // The workflow descriptor type version, must be one supported by this WES instance + string workflow_type_version = 4; + + // OPTIONAL + // A key-value map of arbitrary metadata outside the scope of the workflow_params but useful to track with this workflow request + map key_values = 5; + + // OPTIONAL + // The workflow CWL or WDL document, must provide either this or workflow_descriptor + string workflow_url = 6; +} + +// Request listing of jobs tracked by server +message workflow_list_request { + // OPTIONAL + // Number of workflows to return in a page. + uint32 page_size = 1; + + // OPTIONAL + // Token to use to indicate where to start getting results. If unspecified, returns the first + // page of results. + string page_token = 2; + + // OPTIONAL + // For each key, if the key's value is empty string then match workflows that are tagged with + // this key regardless of value + string key_value_search = 3; +} + + +// The service will return a workflow_list_response when receiving a successful workflow_list_request. +message workflow_list_response { + // A list of workflows that the service has executed or is executing. + repeated workflow_desc workflows = 1; + + // A token, which when provided in a workflow_list_request, allows one to retrieve the next page + // of results. + string next_page_token = 2; +} + +// The service info request is a simple GET request and doesn't take any parameters. +message service_info_request {} + +// A message containing useful information about the running service, including supported versions and +// default settings. +message service_info { + // A map with keys as the workflow format type name (currently only CWL and WDL are used + // although a service may support others) and value is a workflow_type_version object which + // simply contains an array of one or more version strings + map workflow_type_versions = 1; + + // The version(s) of the WES schema supported by this service + repeated string supported_wes_versions = 2; + + // The filesystem protocols supported by this service, currently these may include common + // protocols such as 'http', 'https', 'sftp', 's3', 'gs', 'file', 'synapse', or others as + // supported by this service. + repeated string supported_filesystem_protocols = 3; + + //The engine(s) used by this WES service, key is engine name e.g. Cromwell and value is version + map engine_versions = 4; + + //The system statistics, key is the statistic, value is the count of workflows in that state. + // See the State enum for the possible keys. + map system_state_counts = 5; + + // A key-value map of arbitrary, extended metadata outside the scope of the above but useful + // to report back + map key_values = 6; +} + +// Small description of workflows, returned by server during listing +message workflow_desc { + // REQUIRED + string workflow_id = 1; + + // REQUIRED + state state = 2; +} + +// Available workflow types supported by a given instance of the service. +message workflow_type_version { + // an array of one or more version strings + repeated string workflow_type_version = 1; +} + +message workflow_run_id { + // workflow ID + string workflow_id = 1; +} + +message workflow_status { + // workflow ID + string workflow_id = 1; + // state + state state = 2; +} + +message workflow_log { + // workflow ID + string workflow_id = 1; + + // The original request message used to initiate this execution. + workflow_request request = 2; + + // state + state state = 3; + + // the logs, and other key info like timing and exit code, for the overall run of this workflow + log workflow_log = 4; + + // the logs, and other key info like timing and exit code, for each step in the workflow + repeated log task_logs = 5; + + // the outputs + google.protobuf.Struct outputs = 6; +} + +// Log and other info +message log { + // The task or workflow name + string name = 1; + // The command line that was run + repeated string cmd = 2; + // When the command was executed + string startTime = 3; + // When the command completed + string endTime = 4; + // Sample of stdout (not guaranteed to be entire log) + string stdout = 5; + // Sample of stderr (not guaranteed to be entire log) + string stderr = 6; + // Exit code of the program + int32 exitCode = 7; +} + +// Enumeration of states for a given workflow request +enum state { + Unknown = 0; + Queued = 1; + Running = 2; + Paused = 3; + Complete = 4; + Error = 5; + SystemError = 6; + Canceled = 7; + Initializing = 8; +} \ No newline at end of file From 49c3f723d7d09b0b360dcc0e2d3ac0c38c2a9cba Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Wed, 10 Jan 2018 11:50:09 -0800 Subject: [PATCH 05/15] More proto styling --- proto/workflow_execution_service.proto | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/proto/workflow_execution_service.proto b/proto/workflow_execution_service.proto index b244664a..974796c1 100644 --- a/proto/workflow_execution_service.proto +++ b/proto/workflow_execution_service.proto @@ -180,6 +180,7 @@ message workflow_run_id { message workflow_status { // workflow ID string workflow_id = 1; + // state state state = 2; } @@ -208,16 +209,22 @@ message workflow_log { message log { // The task or workflow name string name = 1; + // The command line that was run repeated string cmd = 2; + // When the command was executed string startTime = 3; + // When the command completed string endTime = 4; + // Sample of stdout (not guaranteed to be entire log) string stdout = 5; + // Sample of stderr (not guaranteed to be entire log) string stderr = 6; + // Exit code of the program int32 exitCode = 7; } From 4bbd503181071edd6a57a2e1d1e71843d8967726 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Thu, 18 Jan 2018 15:13:02 -0800 Subject: [PATCH 06/15] Align state messasge with TES thanks @kellrott --- proto/workflow_execution_service.proto | 56 ++++++++++++++++++++------ 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/proto/workflow_execution_service.proto b/proto/workflow_execution_service.proto index 974796c1..1a710b67 100644 --- a/proto/workflow_execution_service.proto +++ b/proto/workflow_execution_service.proto @@ -163,7 +163,7 @@ message workflow_desc { string workflow_id = 1; // REQUIRED - state state = 2; + State state = 2; } // Available workflow types supported by a given instance of the service. @@ -182,7 +182,7 @@ message workflow_status { string workflow_id = 1; // state - state state = 2; + State state = 2; } message workflow_log { @@ -193,7 +193,7 @@ message workflow_log { workflow_request request = 2; // state - state state = 3; + State state = 3; // the logs, and other key info like timing and exit code, for the overall run of this workflow log workflow_log = 4; @@ -230,14 +230,44 @@ message log { } // Enumeration of states for a given workflow request -enum state { - Unknown = 0; - Queued = 1; - Running = 2; - Paused = 3; - Complete = 4; - Error = 5; - SystemError = 6; - Canceled = 7; - Initializing = 8; +enum State { + + // The state of the task is unknown. + // + // This provides a safe default for messages where this field is missing, + // for example, so that a missing field does not accidentally imply that + // the state is QUEUED. + UNKNOWN = 0; + + // The task is queued. + QUEUED = 1; + + // The task has been assigned to a worker and is currently preparing to run. + // For example, the worker may be turning on, downloading input files, etc. + INITIALIZING = 2; + + // The task is running. Input files are downloaded and the first Executor + // has been started. + RUNNING = 3; + + // The task is paused. + // + // An implementation may have the ability to pause a task, but this is not required. + PAUSED = 4; + + // The task has completed running. Executors have exited without error + // and output files have been successfully uploaded. + COMPLETE = 5; + + // The task encountered an error in one of the Executor processes. Generally, + // this means that an Executor exited with a non-zero exit code. + EXECUTOR_ERROR = 6; + + // The task was stopped due to a system error, but not from an Executor, + // for example an upload failed due to network issues, the worker's ran out + // of disk space, etc. + SYSTEM_ERROR = 7; + + // The task was canceled by the user. + CANCELED = 8; } \ No newline at end of file From 2a73995004e2930b4c6e318c94ab6d292c8ba690 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Thu, 18 Jan 2018 15:15:10 -0800 Subject: [PATCH 07/15] Align tags message with TES key_values has been renamed to tags thanks @kellrott --- proto/workflow_execution_service.proto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/proto/workflow_execution_service.proto b/proto/workflow_execution_service.proto index 1a710b67..13f804a4 100644 --- a/proto/workflow_execution_service.proto +++ b/proto/workflow_execution_service.proto @@ -91,7 +91,7 @@ message workflow_request { // OPTIONAL // A key-value map of arbitrary metadata outside the scope of the workflow_params but useful to track with this workflow request - map key_values = 5; + map tags = 5; // OPTIONAL // The workflow CWL or WDL document, must provide either this or workflow_descriptor @@ -112,7 +112,7 @@ message workflow_list_request { // OPTIONAL // For each key, if the key's value is empty string then match workflows that are tagged with // this key regardless of value - string key_value_search = 3; + string tag_search = 3; } @@ -154,7 +154,7 @@ message service_info { // A key-value map of arbitrary, extended metadata outside the scope of the above but useful // to report back - map key_values = 6; + map tags = 6; } // Small description of workflows, returned by server during listing From 52762ce09bf3dd9d8e2a5e27ccc7fd421c4353c5 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Thu, 18 Jan 2018 15:23:48 -0800 Subject: [PATCH 08/15] Rename messages to camelcase to follow style guidelines thanks @kellrott ! --- proto/workflow_execution_service.proto | 50 +++++++++++++------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/proto/workflow_execution_service.proto b/proto/workflow_execution_service.proto index 13f804a4..09ad5636 100644 --- a/proto/workflow_execution_service.proto +++ b/proto/workflow_execution_service.proto @@ -15,7 +15,7 @@ service WorkflowExecutionService { // Get information about Workflow Execution Service. May include information related (but // not limited to) the workflow descriptor formats, versions supported, the WES API versions supported, and information about general the service availability. - rpc GetServiceInfo(service_info_request) returns (service_info) { + rpc GetServiceInfo(ServiceInfoRequest) returns (ServiceInfo) { option (google.api.http) = { get: "/ga4gh/wes/v1/service-info" }; @@ -30,7 +30,7 @@ service WorkflowExecutionService { // be accomplished by given the credentials associated with a WES service access to a // particular bucket. The details are important for a production system and user on-boarding // but outside the scope of this spec. - rpc RunWorkflow(workflow_request) returns (workflow_run_id) { + rpc RunWorkflow(WorkflowRequest) returns (WorkflowRunId) { option (google.api.http) = { post: "/ga4gh/wes/v1/workflows" body: "*" @@ -38,21 +38,21 @@ service WorkflowExecutionService { } // Get quick status info about a running workflow. - rpc GetWorkflowStatus(workflow_run_id) returns (workflow_status) { + rpc GetWorkflowStatus(WorkflowRunId) returns (WorkflowStatus) { option (google.api.http) = { get: "/ga4gh/wes/v1/workflows/{workflow_id}/status" }; } // Get detailed info about a running workflow. - rpc GetWorkflowLog(workflow_run_id) returns (workflow_log) { + rpc GetWorkflowLog(WorkflowRunId) returns (WorkflowLog) { option (google.api.http) = { get: "/ga4gh/wes/v1/workflows/{workflow_id}" }; } // Cancel a running workflow. - rpc CancelJob(workflow_run_id) returns (workflow_run_id) { + rpc CancelJob(WorkflowRunId) returns (WorkflowRunId) { option (google.api.http) = { delete: "/ga4gh/wes/v1/workflows/{workflow_id}" }; @@ -62,7 +62,7 @@ service WorkflowExecutionService { // There is no guarantee of live updates as the user traverses the pages, the behavior should be // decided (and documented) by each implementation. // To monitor a given execution, use GetWorkflowStatus or GetWorkflowLog. - rpc ListWorkflows(workflow_list_request) returns (workflow_list_response) { + rpc ListWorkflows(WorkflowListRequest) returns (WorkflowListResponse) { option (google.api.http) = { get: "/ga4gh/wes/v1/workflows" }; @@ -72,7 +72,7 @@ service WorkflowExecutionService { // To execute a workflow, send a workflow request including all the details needed to begin downloading // and executing a given workflow. -message workflow_request { +message WorkflowRequest { // OPTIONAL // The workflow CWL or WDL document, must provide either this or workflow_url string workflow_descriptor = 1; @@ -99,7 +99,7 @@ message workflow_request { } // Request listing of jobs tracked by server -message workflow_list_request { +message WorkflowListRequest { // OPTIONAL // Number of workflows to return in a page. uint32 page_size = 1; @@ -117,9 +117,9 @@ message workflow_list_request { // The service will return a workflow_list_response when receiving a successful workflow_list_request. -message workflow_list_response { +message WorkflowListResponse { // A list of workflows that the service has executed or is executing. - repeated workflow_desc workflows = 1; + repeated WorkflowDesc workflows = 1; // A token, which when provided in a workflow_list_request, allows one to retrieve the next page // of results. @@ -127,15 +127,15 @@ message workflow_list_response { } // The service info request is a simple GET request and doesn't take any parameters. -message service_info_request {} +message ServiceInfoRequest {} // A message containing useful information about the running service, including supported versions and // default settings. -message service_info { +message ServiceInfo { // A map with keys as the workflow format type name (currently only CWL and WDL are used // although a service may support others) and value is a workflow_type_version object which // simply contains an array of one or more version strings - map workflow_type_versions = 1; + map workflow_type_versions = 1; // The version(s) of the WES schema supported by this service repeated string supported_wes_versions = 2; @@ -158,7 +158,7 @@ message service_info { } // Small description of workflows, returned by server during listing -message workflow_desc { +message WorkflowDesc { // REQUIRED string workflow_id = 1; @@ -167,17 +167,17 @@ message workflow_desc { } // Available workflow types supported by a given instance of the service. -message workflow_type_version { +message WorkflowTypeVersion { // an array of one or more version strings repeated string workflow_type_version = 1; } -message workflow_run_id { +message WorkflowRunId { // workflow ID string workflow_id = 1; } -message workflow_status { +message WorkflowStatus { // workflow ID string workflow_id = 1; @@ -185,28 +185,28 @@ message workflow_status { State state = 2; } -message workflow_log { +message WorkflowLog { // workflow ID string workflow_id = 1; // The original request message used to initiate this execution. - workflow_request request = 2; + WorkflowRequest request = 2; // state State state = 3; // the logs, and other key info like timing and exit code, for the overall run of this workflow - log workflow_log = 4; + Log workflow_log = 4; // the logs, and other key info like timing and exit code, for each step in the workflow - repeated log task_logs = 5; + repeated Log task_logs = 5; // the outputs google.protobuf.Struct outputs = 6; } // Log and other info -message log { +message Log { // The task or workflow name string name = 1; @@ -214,10 +214,10 @@ message log { repeated string cmd = 2; // When the command was executed - string startTime = 3; + string start_time = 3; // When the command completed - string endTime = 4; + string end_time = 4; // Sample of stdout (not guaranteed to be entire log) string stdout = 5; @@ -226,7 +226,7 @@ message log { string stderr = 6; // Exit code of the program - int32 exitCode = 7; + int32 exit_code = 7; } // Enumeration of states for a given workflow request From 8a169d772f4ff15983defae19814f290019c1788 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Thu, 18 Jan 2018 15:49:58 -0800 Subject: [PATCH 09/15] Add script to edit move/edit swagger and change directory Alter readme to reflect change and remove reference to git submodule --- README.md | 13 +- swagger/proto/workflow_execution.swagger.json | 463 ----------------- .../workflow_execution_service.swagger.json | 484 ++++++++++++++++++ tools/prepare_openapi.sh | 6 + 4 files changed, 492 insertions(+), 474 deletions(-) delete mode 100644 swagger/proto/workflow_execution.swagger.json create mode 100644 swagger/workflow_execution_service.swagger.json create mode 100644 tools/prepare_openapi.sh diff --git a/README.md b/README.md index da37c87e..a195d3c4 100644 --- a/README.md +++ b/README.md @@ -68,21 +68,12 @@ Make sure you have Docker installed for your platform and the `cwltool`. pip install setuptools==28.8.0 pip install cwl-runner cwltool==1.0.20161114152756 schema-salad==1.18.20161005190847 avro==1.8.1 -Make sure you have the [submodule](http://stackoverflow.com/questions/3939055/submodules-files-are-not-checked-out) checked out: - - git submodule update --init --recursive - You can generate the [Swagger](http://swagger.io/) YAML from the Protocol Buffers: cwltool CWLFile + sh tools/prepare_openapi.sh -Find the output in `workflow_execution.swagger.json` and this can be loaded in the [Swagger editor](http://swagger.io/swagger-editor/). Use the GitHub raw feature to generate a URL you can load. - -When you're happy with the changes, checkin this file: - - mv workflow_execution.swagger.json swagger/proto/ - -And commit your changes. +The resulting OpenAPI description will be in the `swagger` directory. How to contribute changes ------------------------- diff --git a/swagger/proto/workflow_execution.swagger.json b/swagger/proto/workflow_execution.swagger.json deleted file mode 100644 index a2b1235b..00000000 --- a/swagger/proto/workflow_execution.swagger.json +++ /dev/null @@ -1,463 +0,0 @@ -{ - "swagger": "2.0", - "info": { - "title": "workflow_execution.proto", - "version": "version not set" - }, - "schemes": [ - "http", - "https" - ], - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "paths": { - "/ga4gh/wes/v1/service-info": { - "get": { - "summary": "Get information about Workflow Execution Service. May include information related (but not limited to) the workflow descriptor formats, versions supported, the WES API versions supported, and information about general the service availability.", - "operationId": "GetServiceInfo", - "responses": { - "200": { - "description": "", - "schema": { - "$ref": "#/definitions/ga4gh_wes_service_info" - } - } - }, - "tags": [ - "WorkflowExecutionService" - ] - } - }, - "/ga4gh/wes/v1/workflows": { - "get": { - "summary": "List the workflows, this endpoint will list the workflows in order of oldest to newest. There is no guarantee of live updates as the user traverses the pages, the behavior should be decided (and documented) by each implementation.", - "operationId": "ListWorkflows", - "responses": { - "200": { - "description": "", - "schema": { - "$ref": "#/definitions/ga4gh_wes_workflow_list_response" - } - } - }, - "parameters": [ - { - "name": "page_size", - "description": "OPTIONAL\nNumber of workflows to return at once. Defaults to 256, and max is 2048.", - "in": "query", - "required": false, - "type": "integer", - "format": "int64" - }, - { - "name": "page_token", - "description": "OPTIONAL\nToken to use to indicate where to start getting results. If unspecified, returns the first page of results.", - "in": "query", - "required": false, - "type": "string" - }, - { - "name": "key_value_search", - "description": "OPTIONAL\nFor each key, if the key's value is empty string then match workflows that are tagged with this key regardless of value.", - "in": "query", - "required": false, - "type": "string" - } - ], - "tags": [ - "WorkflowExecutionService" - ] - }, - "post": { - "summary": "Run a workflow, this endpoint will allow you to create a new workflow request and retrieve its tracking ID to monitor its progress. An important assumption in this endpoint is that the workflow_params JSON will include parameterizations along with input and output files. The latter two may be on S3, Google object storage, local filesystems, etc. This specification makes no distinction. However, it is assumed that the submitter is using URLs that this system both understands and can access. For Amazon S3, this could be accomplished by given the credentials associated with a WES service access to a particular bucket. The details are important for a production system and user on-boarding but outside the scope of this spec.", - "operationId": "RunWorkflow", - "responses": { - "200": { - "description": "", - "schema": { - "$ref": "#/definitions/ga4gh_wes_workflow_run_id" - } - } - }, - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/ga4gh_wes_workflow_request" - } - } - ], - "tags": [ - "WorkflowExecutionService" - ] - } - }, - "/ga4gh/wes/v1/workflows/{workflow_id}": { - "get": { - "summary": "Get detailed info about a running workflow", - "operationId": "GetWorkflowLog", - "responses": { - "200": { - "description": "", - "schema": { - "$ref": "#/definitions/ga4gh_wes_workflow_log" - } - } - }, - "parameters": [ - { - "name": "workflow_id", - "in": "path", - "required": true, - "type": "string" - } - ], - "tags": [ - "WorkflowExecutionService" - ] - }, - "delete": { - "summary": "Cancel a running workflow", - "operationId": "CancelJob", - "responses": { - "200": { - "description": "", - "schema": { - "$ref": "#/definitions/ga4gh_wes_workflow_run_id" - } - } - }, - "parameters": [ - { - "name": "workflow_id", - "in": "path", - "required": true, - "type": "string" - } - ], - "tags": [ - "WorkflowExecutionService" - ] - } - }, - "/ga4gh/wes/v1/workflows/{workflow_id}/status": { - "get": { - "summary": "Get quick status info about a running workflow", - "operationId": "GetWorkflowStatus", - "responses": { - "200": { - "description": "", - "schema": { - "$ref": "#/definitions/ga4gh_wes_workflow_status" - } - } - }, - "parameters": [ - { - "name": "workflow_id", - "in": "path", - "required": true, - "type": "string" - } - ], - "tags": [ - "WorkflowExecutionService" - ] - } - } - }, - "definitions": { - "ga4gh_wes_log": { - "type": "object", - "properties": { - "name": { - "type": "string", - "title": "The task or workflow name" - }, - "cmd": { - "type": "array", - "items": { - "type": "string" - }, - "title": "The command line that was run" - }, - "startTime": { - "type": "string", - "title": "When the command was executed" - }, - "endTime": { - "type": "string", - "title": "When the command completed" - }, - "stdout": { - "type": "string", - "title": "Sample of stdout (not guaranteed to be entire log)" - }, - "stderr": { - "type": "string", - "title": "Sample of stderr (not guaranteed to be entire log)" - }, - "exitCode": { - "type": "integer", - "format": "int32", - "title": "Exit code of the program" - } - }, - "title": "Log and other info" - }, - "ga4gh_wes_parameter": { - "type": "object", - "properties": { - "name": { - "type": "string", - "title": "REQUIRED\nname of the parameter" - }, - "value": { - "type": "string", - "title": "OPTIONAL\nValue" - }, - "location": { - "type": "string", - "title": "REQUIRED\nlocation in long term storage, is a url specific to the implementing\nsystem. For example s3://my-object-store/file1 or gs://my-bucket/file2 or\nfile:///path/to/my/file" - }, - "type": { - "$ref": "#/definitions/ga4gh_wes_parameter_types", - "title": "REQUIRED\nType of data, \"Parameter\", \"File\" or \"Directory\"\nif used for an output all the files in the directory\nwill be copied to the storage location" - } - }, - "description": "Parameters for workflows or tasks, these are either output parameters, files, or directories, the latter two are stagged to an object store or something similar for hand back to the caller." - }, - "ga4gh_wes_parameter_types": { - "type": "string", - "enum": [ - "Directory", - "File", - "Parameter" - ], - "default": "Directory", - "title": "Enum for parameter types" - }, - "ga4gh_wes_service_info": { - "type": "object", - "properties": { - "workflow_type_versions": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/ga4gh_wes_workflow_type_version" - }, - "title": "A map with keys as the workflow format type name (currently only CWL and WDL are used although a service may support others) and value is a workflow_type_version object which simply contains an array of one or more version strings" - }, - "supported_wes_versions": { - "type": "array", - "items": { - "type": "string" - }, - "title": "The version(s) of the WES schema supported by this service" - }, - "supported_filesystem_protocols": { - "type": "array", - "items": { - "type": "string" - }, - "description": "The filesystem protocols supported by this service, currently these may include common protocols such as 'http', 'https', 'sftp', 's3', 'gs', 'file', 'synapse', or others as supported by this service." - }, - "engine_versions": { - "type": "object", - "additionalProperties": { - "type": "string" - }, - "title": "The engine(s) used by this WES service, key is engine name e.g. Cromwell and value is version" - }, - "system_state_counts": { - "type": "object", - "additionalProperties": { - "type": "integer", - "format": "int64" - }, - "description": "The system statistics, key is the statistic, value is the count of workflows in that state. See the State enum for the possible keys." - }, - "key_values": { - "type": "object", - "additionalProperties": { - "type": "string" - }, - "title": "a key-value map of arbitrary, extended metadata outside the scope of the above but useful to report back" - } - }, - "description": "." - }, - "ga4gh_wes_service_info_request": { - "type": "object", - "title": "Blank request message for service request" - }, - "ga4gh_wes_state": { - "type": "string", - "enum": [ - "Unknown", - "Queued", - "Running", - "Paused", - "Complete", - "Error", - "SystemError", - "Canceled", - "Initializing" - ], - "default": "Unknown", - "title": "Enum for states" - }, - "ga4gh_wes_workflow_desc": { - "type": "object", - "properties": { - "workflow_id": { - "type": "string", - "title": "REQUIRED" - }, - "state": { - "$ref": "#/definitions/ga4gh_wes_state", - "title": "REQUIRED" - } - }, - "title": "Small description of workflows, returned by server during listing" - }, - "ga4gh_wes_workflow_list_request": { - "type": "object", - "properties": { - "page_size": { - "type": "integer", - "format": "int64", - "description": "OPTIONAL\nNumber of workflows to return at once. Defaults to 256, and max is 2048." - }, - "page_token": { - "type": "string", - "description": "OPTIONAL\nToken to use to indicate where to start getting results. If unspecified, returns the first page of results." - }, - "key_value_search": { - "type": "string", - "title": "OPTIONAL\nFor each key, if the key's value is empty string then match workflows that are tagged with this key regardless of value" - } - }, - "title": "Request listing of jobs tracked by server" - }, - "ga4gh_wes_workflow_list_response": { - "type": "object", - "properties": { - "workflows": { - "type": "array", - "items": { - "$ref": "#/definitions/ga4gh_wes_workflow_desc" - } - }, - "next_page_token": { - "type": "string" - } - }, - "title": "Return envelope for workflow listing" - }, - "ga4gh_wes_workflow_log": { - "type": "object", - "properties": { - "workflow_id": { - "type": "string", - "title": "workflow ID" - }, - "request": { - "$ref": "#/definitions/ga4gh_wes_workflow_request", - "title": "the original request object" - }, - "state": { - "$ref": "#/definitions/ga4gh_wes_state", - "title": "state" - }, - "workflow_log": { - "$ref": "#/definitions/ga4gh_wes_log", - "title": "the logs, and other key info like timing and exit code, for the overall run of this workflow" - }, - "task_logs": { - "type": "array", - "items": { - "$ref": "#/definitions/ga4gh_wes_log" - }, - "title": "the logs, and other key info like timing and exit code, for each step in the workflow" - }, - "outputs": { - "type": "array", - "items": { - "$ref": "#/definitions/ga4gh_wes_parameter" - }, - "title": "the outputs" - } - } - }, - "ga4gh_wes_workflow_request": { - "type": "object", - "properties": { - "workflow_descriptor": { - "type": "string", - "title": "REQUIRED\nthe workflow CWL or WDL document" - }, - "workflow_params": { - "type": "string", - "title": "REQUIRED\nthe workflow parameterization document (typically a JSON file), includes all parameterizations for the workflow including input and output file locations" - }, - "workflow_type": { - "type": "string", - "title": "REQUIRED\nthe workflow descriptor type, must be \"CWL\" or \"WDL\" currently (or another alternative supported by this WES instance)" - }, - "workflow_type_version": { - "type": "string", - "title": "REQUIRED\nthe workflow descriptor type version, must be one supported by this WES instance" - }, - "key_values": { - "type": "object", - "additionalProperties": { - "type": "string" - }, - "title": "OPTIONAL\na key-value map of arbitrary metadata outside the scope of the workflow_params but useful to track with this workflow request" - } - }, - "title": "workflow request object" - }, - "ga4gh_wes_workflow_run_id": { - "type": "object", - "properties": { - "workflow_id": { - "type": "string", - "title": "workflow ID" - } - } - }, - "ga4gh_wes_workflow_status": { - "type": "object", - "properties": { - "workflow_id": { - "type": "string", - "title": "workflow ID" - }, - "state": { - "$ref": "#/definitions/ga4gh_wes_state", - "title": "state" - } - } - }, - "ga4gh_wes_workflow_type_version": { - "type": "object", - "properties": { - "workflow_type_version": { - "type": "array", - "items": { - "type": "string" - }, - "title": "an array of one or more version strings" - } - }, - "title": "available workflow types supported by this WES" - } - } -} diff --git a/swagger/workflow_execution_service.swagger.json b/swagger/workflow_execution_service.swagger.json new file mode 100644 index 00000000..858cad4a --- /dev/null +++ b/swagger/workflow_execution_service.swagger.json @@ -0,0 +1,484 @@ +{ + "basePath": "/", "swagger": "2.0", + "info": { + "title": "workflow_execution_service.proto", + "version": "version not set" + }, + "schemes": [ + "http", + "https" + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "paths": { + "/ga4gh/wes/v1/service-info": { + "get": { + "summary": "Get information about Workflow Execution Service. May include information related (but\nnot limited to) the workflow descriptor formats, versions supported, the WES API versions supported, and information about general the service availability.", + "x-swagger-router-controller": "ga4gh.wes.server", + "operationId": "GetServiceInfo", + "responses": { + "200": { + "description": "", + "schema": { + "$ref": "#/definitions/ga4gh_wes_ServiceInfo" + } + } + }, + "tags": [ + "WorkflowExecutionService" + ] + } + }, + "/ga4gh/wes/v1/workflows": { + "get": { + "summary": "List the workflows, this endpoint will list the workflows in order of oldest to newest.\nThere is no guarantee of live updates as the user traverses the pages, the behavior should be\ndecided (and documented) by each implementation.\nTo monitor a given execution, use GetWorkflowStatus or GetWorkflowLog.", + "x-swagger-router-controller": "ga4gh.wes.server", + "operationId": "ListWorkflows", + "responses": { + "200": { + "description": "", + "schema": { + "$ref": "#/definitions/ga4gh_wes_WorkflowListResponse" + } + } + }, + "parameters": [ + { + "name": "page_size", + "description": "OPTIONAL\nNumber of workflows to return in a page.", + "in": "query", + "required": false, + "type": "integer", + "format": "int64" + }, + { + "name": "page_token", + "description": "OPTIONAL\nToken to use to indicate where to start getting results. If unspecified, returns the first\npage of results.", + "in": "query", + "required": false, + "type": "string" + }, + { + "name": "tag_search", + "description": "OPTIONAL\nFor each key, if the key's value is empty string then match workflows that are tagged with\nthis key regardless of value.", + "in": "query", + "required": false, + "type": "string" + } + ], + "tags": [ + "WorkflowExecutionService" + ] + }, + "post": { + "summary": "Run a workflow, this endpoint will allow you to create a new workflow request and\nretrieve its tracking ID to monitor its progress. An important assumption in this\nendpoint is that the workflow_params JSON will include parameterizations along with\ninput and output files. The latter two may be on S3, Google object storage, local filesystems,\netc. This specification makes no distinction. However, it is assumed that the submitter\nis using URLs that this system both understands and can access. For Amazon S3, this could\nbe accomplished by given the credentials associated with a WES service access to a\nparticular bucket. The details are important for a production system and user on-boarding\nbut outside the scope of this spec.", + "x-swagger-router-controller": "ga4gh.wes.server", + "operationId": "RunWorkflow", + "responses": { + "200": { + "description": "", + "schema": { + "$ref": "#/definitions/ga4gh_wes_WorkflowRunId" + } + } + }, + "parameters": [ + { + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/ga4gh_wes_WorkflowRequest" + } + } + ], + "tags": [ + "WorkflowExecutionService" + ] + } + }, + "/ga4gh/wes/v1/workflows/{workflow_id}": { + "get": { + "summary": "Get detailed info about a running workflow.", + "x-swagger-router-controller": "ga4gh.wes.server", + "operationId": "GetWorkflowLog", + "responses": { + "200": { + "description": "", + "schema": { + "$ref": "#/definitions/ga4gh_wes_WorkflowLog" + } + } + }, + "parameters": [ + { + "name": "workflow_id", + "in": "path", + "required": true, + "type": "string" + } + ], + "tags": [ + "WorkflowExecutionService" + ] + }, + "delete": { + "summary": "Cancel a running workflow.", + "x-swagger-router-controller": "ga4gh.wes.server", + "operationId": "CancelJob", + "responses": { + "200": { + "description": "", + "schema": { + "$ref": "#/definitions/ga4gh_wes_WorkflowRunId" + } + } + }, + "parameters": [ + { + "name": "workflow_id", + "in": "path", + "required": true, + "type": "string" + } + ], + "tags": [ + "WorkflowExecutionService" + ] + } + }, + "/ga4gh/wes/v1/workflows/{workflow_id}/status": { + "get": { + "summary": "Get quick status info about a running workflow.", + "x-swagger-router-controller": "ga4gh.wes.server", + "operationId": "GetWorkflowStatus", + "responses": { + "200": { + "description": "", + "schema": { + "$ref": "#/definitions/ga4gh_wes_WorkflowStatus" + } + } + }, + "parameters": [ + { + "name": "workflow_id", + "in": "path", + "required": true, + "type": "string" + } + ], + "tags": [ + "WorkflowExecutionService" + ] + } + } + }, + "definitions": { + "ga4gh_wes_Log": { + "type": "object", + "properties": { + "name": { + "type": "string", + "title": "The task or workflow name" + }, + "cmd": { + "type": "array", + "items": { + "type": "string" + }, + "title": "The command line that was run" + }, + "start_time": { + "type": "string", + "title": "When the command was executed" + }, + "end_time": { + "type": "string", + "title": "When the command completed" + }, + "stdout": { + "type": "string", + "title": "Sample of stdout (not guaranteed to be entire log)" + }, + "stderr": { + "type": "string", + "title": "Sample of stderr (not guaranteed to be entire log)" + }, + "exit_code": { + "type": "integer", + "format": "int32", + "title": "Exit code of the program" + } + }, + "title": "Log and other info" + }, + "ga4gh_wes_ServiceInfo": { + "type": "object", + "properties": { + "workflow_type_versions": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/ga4gh_wes_WorkflowTypeVersion" + }, + "title": "A map with keys as the workflow format type name (currently only CWL and WDL are used\nalthough a service may support others) and value is a workflow_type_version object which\nsimply contains an array of one or more version strings" + }, + "supported_wes_versions": { + "type": "array", + "items": { + "type": "string" + }, + "title": "The version(s) of the WES schema supported by this service" + }, + "supported_filesystem_protocols": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The filesystem protocols supported by this service, currently these may include common\nprotocols such as 'http', 'https', 'sftp', 's3', 'gs', 'file', 'synapse', or others as\nsupported by this service." + }, + "engine_versions": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "title": "The engine(s) used by this WES service, key is engine name e.g. Cromwell and value is version" + }, + "system_state_counts": { + "type": "object", + "additionalProperties": { + "type": "integer", + "format": "int64" + }, + "description": "The system statistics, key is the statistic, value is the count of workflows in that state.\nSee the State enum for the possible keys." + }, + "tags": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "title": "A key-value map of arbitrary, extended metadata outside the scope of the above but useful\nto report back" + } + }, + "description": "A message containing useful information about the running service, including supported versions and\ndefault settings." + }, + "ga4gh_wes_State": { + "type": "string", + "enum": [ + "UNKNOWN", + "QUEUED", + "INITIALIZING", + "RUNNING", + "PAUSED", + "COMPLETE", + "EXECUTOR_ERROR", + "SYSTEM_ERROR", + "CANCELED" + ], + "default": "UNKNOWN", + "description": "- UNKNOWN: The state of the task is unknown.\n\nThis provides a safe default for messages where this field is missing,\nfor example, so that a missing field does not accidentally imply that\nthe state is QUEUED.\n - QUEUED: The task is queued.\n - INITIALIZING: The task has been assigned to a worker and is currently preparing to run.\nFor example, the worker may be turning on, downloading input files, etc.\n - RUNNING: The task is running. Input files are downloaded and the first Executor\nhas been started.\n - PAUSED: The task is paused.\n\nAn implementation may have the ability to pause a task, but this is not required.\n - COMPLETE: The task has completed running. Executors have exited without error\nand output files have been successfully uploaded.\n - EXECUTOR_ERROR: The task encountered an error in one of the Executor processes. Generally,\nthis means that an Executor exited with a non-zero exit code.\n - SYSTEM_ERROR: The task was stopped due to a system error, but not from an Executor,\nfor example an upload failed due to network issues, the worker's ran out\nof disk space, etc.\n - CANCELED: The task was canceled by the user.", + "title": "Enumeration of states for a given workflow request" + }, + "ga4gh_wes_WorkflowDesc": { + "type": "object", + "properties": { + "workflow_id": { + "type": "string", + "title": "REQUIRED" + }, + "state": { + "$ref": "#/definitions/ga4gh_wes_State", + "title": "REQUIRED" + } + }, + "title": "Small description of workflows, returned by server during listing" + }, + "ga4gh_wes_WorkflowListResponse": { + "type": "object", + "properties": { + "workflows": { + "type": "array", + "items": { + "$ref": "#/definitions/ga4gh_wes_WorkflowDesc" + }, + "description": "A list of workflows that the service has executed or is executing." + }, + "next_page_token": { + "type": "string", + "description": "A token, which when provided in a workflow_list_request, allows one to retrieve the next page\nof results." + } + }, + "description": "The service will return a workflow_list_response when receiving a successful workflow_list_request." + }, + "ga4gh_wes_WorkflowLog": { + "type": "object", + "properties": { + "workflow_id": { + "type": "string", + "title": "workflow ID" + }, + "request": { + "$ref": "#/definitions/ga4gh_wes_WorkflowRequest", + "description": "The original request message used to initiate this execution." + }, + "state": { + "$ref": "#/definitions/ga4gh_wes_State", + "title": "state" + }, + "workflow_log": { + "$ref": "#/definitions/ga4gh_wes_Log", + "title": "the logs, and other key info like timing and exit code, for the overall run of this workflow" + }, + "task_logs": { + "type": "array", + "items": { + "$ref": "#/definitions/ga4gh_wes_Log" + }, + "title": "the logs, and other key info like timing and exit code, for each step in the workflow" + }, + "outputs": { + "$ref": "#/definitions/protobufStruct", + "title": "the outputs" + } + } + }, + "ga4gh_wes_WorkflowRequest": { + "type": "object", + "properties": { + "workflow_descriptor": { + "type": "string", + "title": "OPTIONAL\nThe workflow CWL or WDL document, must provide either this or workflow_url" + }, + "workflow_params": { + "type": "string", + "title": "REQUIRED\nThe workflow parameterization document (typically a JSON string), includes all parameterizations for\nthe workflow including input and output file locations" + }, + "workflow_type": { + "type": "string", + "title": "REQUIRED\nThe workflow descriptor type, must be \"CWL\" or \"WDL\" currently (or another alternative supported by this WES instance)" + }, + "workflow_type_version": { + "type": "string", + "title": "REQUIRED\nThe workflow descriptor type version, must be one supported by this WES instance" + }, + "tags": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "title": "OPTIONAL\nA key-value map of arbitrary metadata outside the scope of the workflow_params but useful to track with this workflow request" + }, + "workflow_url": { + "type": "string", + "title": "OPTIONAL\nThe workflow CWL or WDL document, must provide either this or workflow_descriptor" + } + }, + "description": "To execute a workflow, send a workflow request including all the details needed to begin downloading\nand executing a given workflow." + }, + "ga4gh_wes_WorkflowRunId": { + "type": "object", + "properties": { + "workflow_id": { + "type": "string", + "title": "workflow ID" + } + } + }, + "ga4gh_wes_WorkflowStatus": { + "type": "object", + "properties": { + "workflow_id": { + "type": "string", + "title": "workflow ID" + }, + "state": { + "$ref": "#/definitions/ga4gh_wes_State", + "title": "state" + } + } + }, + "ga4gh_wes_WorkflowTypeVersion": { + "type": "object", + "properties": { + "workflow_type_version": { + "type": "array", + "items": { + "type": "string" + }, + "title": "an array of one or more version strings" + } + }, + "description": "Available workflow types supported by a given instance of the service." + }, + "protobufListValue": { + "type": "object", + "properties": { + "values": { + "type": "array", + "items": { + "$ref": "#/definitions/protobufValue" + }, + "description": "Repeated field of dynamically typed values." + } + }, + "description": "`ListValue` is a wrapper around a repeated field of values.\n\nThe JSON representation for `ListValue` is JSON array." + }, + "protobufNullValue": { + "type": "string", + "enum": [ + "NULL_VALUE" + ], + "default": "NULL_VALUE", + "description": "`NullValue` is a singleton enumeration to represent the null value for the\n`Value` type union.\n\n The JSON representation for `NullValue` is JSON `null`.\n\n - NULL_VALUE: Null value." + }, + "protobufStruct": { + "type": "object", + "properties": { + "fields": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/protobufValue" + }, + "description": "Unordered map of dynamically typed values." + } + }, + "description": "`Struct` represents a structured data value, consisting of fields\nwhich map to dynamically typed values. In some languages, `Struct`\nmight be supported by a native representation. For example, in\nscripting languages like JS a struct is represented as an\nobject. The details of that representation are described together\nwith the proto support for the language.\n\nThe JSON representation for `Struct` is JSON object." + }, + "protobufValue": { + "type": "object", + "properties": { + "null_value": { + "$ref": "#/definitions/protobufNullValue", + "description": "Represents a null value." + }, + "number_value": { + "type": "number", + "format": "double", + "description": "Represents a double value." + }, + "string_value": { + "type": "string", + "description": "Represents a string value." + }, + "bool_value": { + "type": "boolean", + "format": "boolean", + "description": "Represents a boolean value." + }, + "struct_value": { + "$ref": "#/definitions/protobufStruct", + "description": "Represents a structured value." + }, + "list_value": { + "$ref": "#/definitions/protobufListValue", + "description": "Represents a repeated `Value`." + } + }, + "description": "`Value` represents a dynamically typed value which can be either\nnull, a number, a string, a boolean, a recursive struct value, or a\nlist of values. A producer of value is expected to set one of that\nvariants, absence of any variant indicates an error.\n\nThe JSON representation for `Value` is JSON value." + } + } +} diff --git a/tools/prepare_openapi.sh b/tools/prepare_openapi.sh new file mode 100644 index 00000000..a206fe19 --- /dev/null +++ b/tools/prepare_openapi.sh @@ -0,0 +1,6 @@ +# Add a line for setting the controller "app" which matches the name of the +# module that hosts the controller functions. +sed -i 's/"operation/"x-swagger-router-controller": "ga4gh.wes.server", \n "operation/g' workflow_execution_service.swagger.json +# Inject a base path +sed -i 's/"swagger"/ "basePath": "\/", "swagger"/g' workflow_execution_service.swagger.json +mv workflow_execution_service.swagger.json swagger/workflow_execution_service.swagger.json From 12ec02533177c15961d368639bae2bb238fb2897 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Mon, 29 Jan 2018 11:26:43 -0800 Subject: [PATCH 10/15] Update readme, remove swagger docs --- README.md | 17 ++++++----------- swagger_editor.png | Bin 11447 -> 0 bytes 2 files changed, 6 insertions(+), 11 deletions(-) delete mode 100644 swagger_editor.png diff --git a/README.md b/README.md index a195d3c4..ab10259c 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,8 @@ -![ga4gh logo](http://genomicsandhealth.org/files/logo_ga.png) +GA4GH Logo Schemas for the Workflow Execution Service (WES) API ==================================================== -This is used by the Data Working Group - Containers and Workflows Task Team - -[View in Swagger](http://editor.swagger.io/#/?import=https://raw.githubusercontent.com/ga4gh/workflow-execution-schemas/0.1.0/swagger/proto/workflow_execution.swagger.json) - The [Global Alliance for Genomics and Health](http://genomicsandhealth.org/) is an international coalition, formed to enable the sharing of genomic and clinical data. @@ -21,9 +17,8 @@ interoperability. The Containers & Workflows working group is an informal, mult What is WES? ============ -This is the home of the Workflow Execution Schema proposal. The Workflow -Execution Schema is a minimal common API describing how a user can submit -workflow requests to workflow execution systems in a standardized ways. +The Workflow Execution Schema is a minimal common API describing how a user can submit +workflow requests to workflow execution systems in standardized ways. Workflow execution engines (SevenBridges, FireCloud, etc) can support this API so users can make workflow requests programmatically, adding the ability to scale up. In addition, these workflow services could have (and probably do have) UIs that would (possibly) use this API under the hood to facilitate workflow execution requests. @@ -54,9 +49,9 @@ Outstanding questions: How to view ------------ -See the swagger editor to view our [schema in progress](http://editor.swagger.io/#/?import=https://raw.githubusercontent.com/ga4gh/workflow-execution-schemas/develop/src/main/resources/swagger/ga4gh-tool-discovery.yaml). +The file `swagger/workflow_execution_service.swagger.json` contains the API description. -If the current schema fails to validate, visit [debugging](http://online.swagger.io/validator/debug?url=https://raw.githubusercontent.com/ga4gh/workflow-execution-schemas/develop/src/main/resources/swagger/ga4gh-tool-discovery.yaml) +Please visit http://ga4gh.github.io/workflow-execution-schemas to view this document in Swagger UI. Building Documents ------------------ @@ -68,7 +63,7 @@ Make sure you have Docker installed for your platform and the `cwltool`. pip install setuptools==28.8.0 pip install cwl-runner cwltool==1.0.20161114152756 schema-salad==1.18.20161005190847 avro==1.8.1 -You can generate the [Swagger](http://swagger.io/) YAML from the Protocol Buffers: +You can generate the [Swagger](http://swagger.io/) from the Protocol Buffers: cwltool CWLFile sh tools/prepare_openapi.sh diff --git a/swagger_editor.png b/swagger_editor.png deleted file mode 100644 index e35ad5d54519e918c7c18721ae0cec8245ddb9f1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11447 zcmV;oEJ)LdP)VwGMV1zJLhWHAd}>tbKA`1{{6)i?kO{KpL^O{)Uj9$WW>Skv+2CS_B@3` znX6XnN>nOsrb?kUDU~XX0+dPx`UiX+`b?rRh+@K#SjZEJ1nrS%pf(zd*o@lDMzbNu zFZFBbD=1`v@QcUaY6*lq)AU;FFpWxY@CBWwK*%**qc#lHstoxml}4vjsMRW^Mumi- zKp#ObH8N6(MWVq-G!_m=qM<-2>}v@|djBh`I~?}=e4&mCo!YeCq|fdU<%RYU zWP$L&zM9W2v>FTUvKsQoSd4j@N`*?2ejpj4QL3fG>1*EO^fvx$c=5UKNz#!NLKX-i ztQAzJ<%^P>9dAw)kclf{fY+~X+oZW{U8$Tr)V^@q3PY0EX#oPQuiyF zkOk5UZLVX}O}ea!Zolmeiy?oOPGizZ-6t*t!``6F*SyZG&spjAw{Az`UTrbtO5Lr5 zMHa}GJ|P+O*-B z{<#xxS1Q%LZ7209@PfbA_WgfLb0=OWbzh{UPYR^nU3*TNsc@n%(0&t|QWQwtpG3wR zu(vwA4KEtC8QaoLMITGuAF=6E0zn`4Eod6?8;c=tpwxXyM%;c|ol2qkyFr)!PpSJN z7AY@~HrKJqndZ_tfsp$yz1Es0b$?O;^j^B{?&=rx8q1d%=8~_a?vIe9tU%gaRg=)7 z;*m^C|5;M^C6#e_>NcT!=>80I@i$WUMF>(-ATD3?4Q4~mOA1g*-IYGW;l}Uo9$YZv zNvZq6Hz_3$x3A^ua5Qv(nz5k#C%Zox(TIW5+gz(sY5r`|WxXPGZ}=i51QHB+#`}Vf z=h93?JDig`m}W2!iNuZl4W&!FKS6T;TDyfy>_lUi2NGIm83M zCX+53&~y=f&jP*H3I=T&`aKu4Y9nM@211r)09cI$fELTl-x7xydTs5?Jb|`{bJEK8 zNZlJ+l2Rb(=E_qlRR1yP(&tIpk8mUaHMZ~JP~#frhHrHpg_@Ra;Pf?0(jT)v2TF4% zGQvR*4j5cG6Z)gyH7dO%{pkcf5I&1W!?zlB8P$?@go30Lh|AZ~WHIFCN!5NR;)h+w zK7e=DE=E85r0NDMu<8B$sn5Zi{K5C6GYKGcFYNzJaD3 zL&a_zPnWhISql4WKZnZtUvC zLLg`XJ_IfNewS?>yilc7i?-Nku6b9r?VFWX#|D{lpBi`Ny zT5rYLoJT*g81nvz)>H|e!kw;p^AHdVd?^=-SvXi{?W z(ucw1p^IShkVSyjb)vM1)^DLe$n{H;KKmU}8l)tl1%jqEORUC%g+jLzjs#)b;WyCq z=?mCfy&Ri;3*BTwf_Ke^7oUfwVdufb@`a)-?%h7yyJ*V1K=6hrNmzl@*tS1bl5^U< zLbro=pZ@jPnP@t-Tj-{e3N)RCJ1<-ZR%5>4ZK0{`tA&}Pt{1!^3KC8r{-ATD+i!a! z+d6Pg?-4)fvAb(vbLA5FZtpXAjU!k)$&6NQg40Lb29wJdK~7qkpl#XR)!R(EtebTj z^A159A|#;%g65^u&?4?Dt=cHqiPyn~Z{V}{BgYak?L$gDrbqL0_pHHypm=r+t&i$?0FJ4R=kA$&-tucrWbT-GfWw_ z7|s}VH>8`2c&!_KmY*1PnUWZ$N+J*;#IgPGJMiv?TfiTbV>e5QMrDBc)1HG#L$2es zUL=y`NF;N4t&|0#3j|H8UP9BV8~Ca(5)H%BwKu?)L;vNgN*Nv)-G3fjcji)1tN4r5 zD#dh5R4*g6ApbX?;=p{c=~uEi5|G_hAH&jhH}P60632YBaQ}+eN_#=H5yaMU_^zC^ z!Mr*3+d2-x@~!v6uBs3Cs!~P(%8Sl|izht+g_)yxtJChP`b(yz{~vg(um?mDNMrlH z^9nLXe8yLO6?Lm%$=8>GKjh@AN*MuBE46U_?Dt`0>4m)2+3cuXl$TzcT!Hk!WbLPGjEAONDU|bdm1zH9yLi zIF_!vQ5HuEuzL63;n~k-!~a4Y(W6%BG#-EJzq$9Sdju0mDB`<6C#{T^!<~26-OL>R z$)pCA4Xe=_68lJT7YSOdPinC5{SWtE#Yd1SR)@For8Hv!FWb3qeRn-2Or|V(<)gUeTN^8y${cwwMwf(_uvEEd-Oke z8$kjg_h}lHVH?-;jxTpS44-fRBQJfDNiJrNybCUw%+1&WpO1vY!6_QGeiI*k;%x*8 zhP_L;ieuZMH)L`2VZPn_6s+CHZPykr_(KsdZ;n8mM-a3yN{_~(`z?lCPRTGU>Q=&Y zUr6daCX*oGMLXW%;({;eaHD&BxKWo;%cUO9BZyL|UcyxzEsjcfdDX>SyCRb!ytHy2 z)V6Yq{$tRlo7GC~3taocNg&ORiVV~d&Ug0mZTI3S8rN!MQUn~?aw2TQa&0D!?F#x!tSb%xKtyPf*fvK3vaBsmdU}w1+zZqQ7+}F*C4@= zXPj1J+{R_^`oG^?1>29jO>yC@VYzGR>t>oO2qJ$_)EcE16VoCyE|#9sYUgz;08eB_j_mI0`DI=4XFZ zFw+~HAjsjVh3b~AOe(pm=1ox7wo{UR;(RO{54;FdhTTYAhgzk34t=BY*rFOiT)vje z(Zc*)ng)L?-^|=_Vy`=LtYt8BcWMyO@9sVuD9CuID4@;r1|1Ibwl0y-f=<%>BmDG={-J-MGG367V$Q z?6D8P?dN~PyI>Z!}4z7`&7TK`+pazp~bIRdT5 z{3To#=I^b)1=ZuCE>D75rH8xcZegO5Yt#l&+UyG`m1>wX_CDslO*0mX(q=RkhNbJd zMM)WyKkadcr*0NW86ttW{kA`ob?HPt-~J%%I7+>AA!hMeA3?wDv7)ut*T}S#!opeN zeV+SjK4UTqb3;Ki)nYA>HrKJ!vMl}2Bk$0UV2jN!cRWN}oS0JwEr4Og=ZMx`Uu9ta zG?+g8|3qtVQ{~IBpj2 zgP5$@!|8k)hl^ru^v7Bt{-Em)y6ynfweDmxJBfu!mxbq>qP5r8$+Yx`Z0jJ=+QZ3^ z@JTx@sK2+ey_S_gyaD@-daX5swnHCo{xxk;Dkha*&+MJcbdQ4PWJN+H(&>&5H__W_ zY1Kx9KjgZT&9$sf(j(CjtIFbWp!O@#lkkr!B@Y7L(s2-W99asXun#InEiL= zbQz*4bx4SLdDSJLQfiO?uGd(=q|atjKjZh0baP{3*+RzIO)To#cEHY~%V6xl`Bdf6 z=ZWWqPxp3RuP5p4?wT_)%q7(Gqknun0cvd9Xp2+v^r8^670Cvi6aB@nE`kGfUkco- z(F6#1V=`g9c zYkC!k+uyc`wgVdu2o}$)Th1PPKhgD-b#m6*?jVgvcFF*{=?)h*R?sKIO*i%HHFjH% z0y)yS?h&&-yO^eNoSc8n-d?A8yfdg^I_ql#Auk+jk@KZ@htt~#tmUE{RJ7m~-%HS_W53x}*CtrZ zP9d4xZq3Kh2{A9^c3o=Iv)RXSPJC5u2KSy02iL2Xu8a+(0Vhd;+ z#7-_!kY*w?oQp)cxpVdfLJ_hjGv66**}}#pwFag z39&)Kn7Un0hS`993}kvH6d~JaQK_iI9B}5x95Qr4NIU_1Tub%10!cR)Q^mTfYyAP5 z9rP!gTu~~?20d&-BGWUZF$?&}^+AinQwIkd)=-s2Yp0nmU&~4AKIzr8I*&}ysPsP% zcfkrdC9{mwRMgR>QO8S)aKwN9&;NdsK+O6aYSR*0^~%Wb#op>qsmq%5IU|1lx03{d zhMB1}l{>vnaIAR)O;H&c-e_{{2V2KssQYe(DP=u4v+5ks>#O5b7nDm*acK!VXfrKIf zD!1Bghu);>4F8b1k%pu4L1AwML!=H+{JQee@(qzM%l`<4{W)EKKS3bTXqY&12|kT} zmt#OLzBoA;bI*iC^nOcLoW5qF>s18KYC`hW&WDMu05H*5dawaSY6Rma2d92ZEMsqkK!PT*kFi2})o zgSRg`Jaq#0CUqb&>clx#5(T&4MpZ7;RC06Y&&LH~HRPW{Q(3F?2vsM9K&xV()*vEV z^0LO|XTJ`~P3M#nkXLMuLsaF=hMaz#KQjVBb8(Z!m`AmPRn@YEsuMymFrU2^a}2qYMCPor_K#eTqUe+zLrp(xB8 z&HCC+m9Gfclhoqt-G5_!t<}IOBRH>wBYx`h5WU7a6Mts}qSvMo?_Oa@cQ}N^sLepH zgI&M?{8U-J`*DGLlse$VHEc72DU~W1kULr697!Oti#(# z)wmFhC}Dqu;}J_z8niUJdj1j+LeQhG0 z-6RXOtvgs>JAUxhOfc+3fc6*y&8nC@oIg&n)W%^nl>`Yx!U)A#WA0(S{=jTa7~8Sr z0o&WLJut-Ukn_fg7wNqxd_g*a=+q`=N}_c8Y#OcFkWW*I%P0K~*1)`JaKVH}Szmvk z?km*qCINets>INP<#pCq<9zgk^4W)h6$;vQvRb9LHajW`l-{7dP^HvR`GEL?4ywla zBHdIBzq()zQJ?^vuSAB9{Wa{*Lohh?Hcs>@CH|n3svOdUMyXU6s1=~fRw~p~1~k49 zUE^FCRdNwbD8CNI^d}3xf=#zKRFFrUH|w)u9G+L>X)40P=(DH#6PoUG!biCVL48-L6xt$%-Bo?D)mZo;jhXUa-X8^x?R%*@ zz?S(_pM~2!9_wq5<%zP8Zm*MniC;Dk%Z%P~iW-4&8I}-D9%0&em!?HlY6hbHcuIF8QhJoQBYVRSd9ft zBL42gfhjMV1$WQe0y);f#PwtDxNsd32M#-XvJ(@ICOEwKT)69^4KN^Y3hO#6>sP_% z$|Y3&!HajC<_#zO%uO2t1)1FjRp$*CW3Wi)T8%z;BN7N2->HZ)x%`rWRX;oiFMf3) zQJL$`e4CYjNm14qxO&EG#C^JQ`iqdpBG5BVfp+B#aUlre7@nMYGA`nZ3Ps z5vBu8b_h zEB0F!oR+Ecj zFeIfE{BZ08G{=6BxE#)NNCMtPyLb|NALt1~x=lyXEu%~4vA*`yL06*DkIeTim*9cW z3ASh3#32QmSkMMjS73=Ks1u4Ou0U{%)>l_AfF)mFMpZs9V>odclAGRQAWPGLAvp*= zf80aF^_LgTCN6_{XWe3W{)=;{O*+JZb_7Y$Z2Q;)j!XJC8qLhpZ;bLx$T- zh8fxg;~yc;osLg@Nu0rU99c$Gzc<(p6?NkjSU8*PCVqAG6|A1ZwvI#a_S)-- z`}OYnn;Dam-oU}nXzE|O?nbC<-P!APc!vGb%DF_@w&qQI0xp^S6mc9fsZc5DvNcCz zksuOC&>p1>P~2%dBqVLlDscIlabi(sF-V&gYV3f^n3Hd|;DC5fFQ3W;AM38oeKnuL zlb=kcTA<=I*-tK?3`crMMO|b2o~fVKoaf2qQyBZH_{32C+u-RH(}^6&@TvQmqkm1+ zAKWH>udo$3PM6W1W2 z3(iA0Ztzvi`eI1onM7r=^UOW3<57=n+b`2x3a1WQ09v(?dE#Qfh>H4E%nil|_z-Xu zV@#OF%m(m+{6OtWBn;BEP<#q9M#9L_3qY;ZG1KBu#1A`uc4Z9iEA9$XqO}o1qzTU+?vIi+Rw%D3IPYG5(9)b8UDjiZdi&z}^eOtcu z9?oskV$_jkKJFwM3q{bqRtFlkN=e0m(QD`g5{ZODXo}UOG#hdPp)j3Uu9f0Mm~ZeDZ{-hvVSW9W0rSvPCGj^G z5|PP67qPx}bLFeBn}2n}s8?Obaa6WN0iNGKej5{Si^IB{69#~@quluwnuOny51$WSzi zYlOmT?A<9X9>Iu@3IERs#2@UST7zV=blw*OoC|`jK_7NE#dbt8?7*(hI+k1dcpF~m zI0Yo+rj=2Z3k2Ov%u+@mT8(KPjZrT?I7=jRlg9S_tgoFq>}CObl3L6f^E=kpIy`i- zaD@OLD$^cC_(RU`@OMTaW_`BH4gq}=P0aio#`sXnO-YhPw`>0H*=R<&#r zuqR}c=1yd03#9mh?f$&FcmHHl#=lgN;K+_wH>yphlwz zNqpBtAsKw?r6w64B6;blu-Cch_;SR~R4%!+Fd#N()k0c1CnJCc2zv^qy z1`C%G(zF|wDnpv_W!F%3N>UPIZy}*clnodXa4r`b(7k_B zBF2*hVlm`=M^kyasUMW&jHf9oBf}dx)-uS?ph~%kgOHq*;pZfQxP7+GRJ8-$wwjC# zpN#6y-Yn+lczmrVC4)anAX(OdYr~NM)oIVDQnDNzGBSLFvyssdjfSIYmF}ctj=0-6 z4o@9>=ZY?1<1m~LPb#8>QB;$0el0enVLNH#mhpkrPyj=V>1HtwZ~aCbpW)|!k1G(P zHsc$rRsfrkvrpW@H`o_}1f?SRc4)*#qcRX(ABjk3YJ!7GkYqgWicyyp_mDhpZBdB6 zf`=TMhNSef@DZk^CQUNa7t!I&xCDfwbO}xQVEXV|smd|I03(682`u9Z#H`Qux_m8F zG>zdzFF1vn6uA9l(TkJ>MtFS{EulbgEMY0Y=@)22{CEQPR+By}?##wS>8yg~M! z!^MQjI{P9d=L*=8mvkM-2fx2`7x7xDllrz@E3&Nvf9UStdtCAl$eVnJ*Kcp8Y1nMY zff*xirzy&bV@(@aU#r(zpeXAU0eh3&;HZQoQ9*DJZn2p<>?Z2SCH|1hRg^X6qMokl zbx+Kw&m;?l-i0X=Z25k@`*EV{aoPcy_)IRp0Z0ti_SUQrt~1>*p^QAGb|)R4`qz59 zrdNTWdHojJ4&dC|I6s4!;K(K<34vyh`5g?%oh)FFl9Qs$F=$>-HUn6>^D%*X6CZpi zJs^)hsty_Fs%4&AQ?T{ z-F5KbJ7n&;T)Fm)w_x1hUr?3x1{^k{F1=TuYgYT5Duw1r^o`1|7EfXD6Z)8l@sf5O z`;b^5XuZXJb;q5rXwk@&81AHefKxZ~Y41;xG?Wi;)+}0aVAM(ttGPIWk4q*$ z4Ws(c<5C3?udQCdB>I;T2AsQo$=8>Q)?RtxY`FG}H-XFM0$O_!-NQrz$+VPiKyM12 zP4Q1GnD!qgOpq9O0&VcbCzIgw?SEqO=E?8_w$a;MxrBLdZBFqXKnqTPiBtJwEt?-j zqsQkYWr)2?%!ZujIy?>3HU&7Ku$U$&-hdsx-0^34c@Um@YfHr6MG)Y z0Aro7^XPl<)beSfpLt-oW&~3zHTTh!V|9`q52v?jo7I?4Z4SHT;A`;KcMEBYb7o-v zG$_tK4YDl*m~cfnY!RON#S12fPtxyys}Q0I6xyFFk6hId*I^95RvZMp!`lG0t=~hd z^C&d6?`JkVqasP=Cm^ zoK_qdPSKk^e%tT4l*f)$SIt-gLyG8P8OfwBMOkCuhO<5d9mnYMZePozT*`A2NN#$0 ztv~3ZilNn&izhxI`-jV%z~xh)<6K85;$MjavU90}bNv5EG`QH~Z{y@=iJumW=X}hy zD>5m<^=RtNHJwHv9d;9R|XQvNpVd{7*TRQdTLNRFUq6{7mR-l zP8oO^r#gJWjz+t?+F;P7b56O-TLu}lXulS35prSL@SEYxQNQ7-M>5IAq#@VC9Lmg4 zUD2rYmltG=;KiPwH-Tu?#w|AIq1UL)hgY3 z9=gOpeGU>2f6zG`b?yiB#rc^0)zw!pC$}=GPkGU7xaHh0IcNNeM8nZ=BrrjzF>U9m zYy6EM7`@iI-yd==;G?dEv)+Z#B3k~+NW-A~(~&s7;w2qkoAbzGKE;9Ibp+|evj1d5_^FNSaq9j$bs`_ptK$ z?Fk&p9v>Es9e4#F-I7Tl%8Sl|+b`fp94#G{%Y+aIMvxJtlQ!3}Z_(mw2KSm`@a*Sj zLuEa+#ehr#k!>9W_s_53C4;WRQ&*90>PMH9kOhJYBoK0sMQ_4pjY`i;kls(W{0>&{ z{yQI4$w)>HnclbkmhcG!#J#^J!YOfjX~dZwM+740-{lD;3J`_)6NpZvXSJ zY{UQYQkRUlTr}ZtoC7X&#p!K6$7;;`inj{mAjle|D=MXWZA-_&Yx$~g`tV!e#yQI+ zi5xFO&%#;n@hXnijw8PlS{xX`N03g?dgLLr9^n-9p(_pcJ@C=yU(1n85*+M|Hh;>$ zp){8)(q&IL+;z{VoBN3}jSx*Bm}=YB<)t|jc+D04=C{|u)T~vW!T)vj27DFy?`?jHop9$}?r}`7Vs+8e@p+$3$I6k0GY!eS} zpnZi=mmzAt3eg3E@%Y<5GU>B;Jtp5&@iHvicpKwSEh8lgxNOQlIR{zzIsTCAOTET& zF0YlcKoWsqJbwGfCVeKac@f&(wXkN-6R@%3B?yMSd{!;f3)tZeZ{eOX`qz+S9n5Rp zQi%g2nLsdnPq#X8dYjc30J3GA;e@;lJh>Y&-NO ztl$5Q_;=5S4>Dk2n5sOICw)&+TfRjc(XA$ z#o9-k?}Dlp>XeHz@S`v;nfw$JuSytV*y-sH+15dS5vTzo5=J1H4sXK@qb_qLU$$HE z*j4o*e75ZYXmacqs)>{YCuBYM)IY(ggRkJljw&9JXfT4-9CIy(yw3z_fP#b)2qqA6 zpXLd){U_60Dp;0(>}P_*X>P1|1vVUbUfvq+20WTi8G0l7x&ex^#)49y(H+y_sXt`U zWn6<@w*_m2f`k)DCyn;KuN7pDq)&p)l6H3uJhgl(Hk%f@siXppw9K>_I#n+!DlA;pb;m+@9VZ*IH>oilDRN#_)Sb|Xrb{1 z9LvMe;5>^VHz9Atgcbx z)S)+l)tE0zn~`WJYIjxLk&{-I(8(kdULY8M&^6lSZ+;}pGT=f!ocq~;*C|JvHp1TO z(iou+*_b~}*%=#S3`rr?_d=7VQUVg@K zmt>uwAV~${7@$(BpG9jENetLJK~F~E;eZGI1E*`t!Si^#!wPx$STd5 z!0bF(^tn))I}uHv&H#fpO_Kg}Vt3bkgZgkA`Z^$KM<_^2fpmhVJ>$@H>t3TS{Sq#z zf%yc-C-w)OU~?X3(wMb$RN{>WJiWjrq0sAvqmj2vQL`g_e3#5~FckLN! zmF{_qAy@o~H3zVmqD;W=81r+dSQPj_1s1%LSv(76ej^kfe`{4Z8oUjOI$=)4X-Nrz zbb{6fH$|eM2a!+;r0z>!1dWvKN~QWiG?n_7)P3QLmhf?=N2vSxc zogkr{?+>{y$TXMyQmZx@r0!2D5DNPO?Vj2XbQ;S?NErNvElDs^ULc*gd@U-CO8=`! zEc|ysZE@tX#v1y=V;2g-Kfi0hJJqAsLQOEx=)FOERY^>`&*5n zuy>A0pFPa&w_#uD)6ha+6u%Uq2uFeum#=xZS)cQPC(wEby`C@7>q{yFLNSpALWtd6 za~^uyjtYnUg=k805qjblt5u0P%HYw#8?ZMh6sk{<5bC^v_5&H_VmXL00kS}NptAn! z-=~}UEwmVNOORkBJ!?U02*1m3YjAk#-b5qbLj@TlB2xAz*^vdpGtG|561~*kkoAYqKMr9bSRT~GRr?^$6)M(HeN2OA#73?jLV`wyp#G*{fu}~-+3AoTA`fw=X z-OUApf9kGAqzxAS{#*y3Z*Jvt<)Bx#eW_Wi2+T2bP5ID zHdA5)UPdu7Y}^Y`^wGp3u~;~W#NmrX12!a%`bad~l$TyEhhPzj{|^8F|Nng;wDG}d Rw9Ehi002ovPDHLkV1k<(PBQ=i From 39d149d6e683a8f939d4d56c4f9a454b01eb127f Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Mon, 29 Jan 2018 11:30:21 -0800 Subject: [PATCH 11/15] Update readme Add requirements txt for installing cwltool Update swagger json to reflect changes --- README.md | 19 +++++++++---------- requirements.txt | 4 ++++ .../workflow_execution_service.swagger.json | 4 ++-- 3 files changed, 15 insertions(+), 12 deletions(-) create mode 100644 requirements.txt diff --git a/README.md b/README.md index ab10259c..b6fc9599 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ Schemas for the Workflow Execution Service (WES) API The [Global Alliance for Genomics and Health](http://genomicsandhealth.org/) is an international coalition, formed to enable the sharing of genomic and clinical data. -Containers and Workflows Task Team ----------------------------------- +Cloud Work Stream +----------------- -The [Data Working Group](http://ga4gh.org/#/) concentrates on data representation, storage, +The [Cloud Work Stream](https://ga4gh/cloud) concentrates on data representation, storage, and analysis, including working with platform development partners and industry leaders to develop standards that will facilitate -interoperability. The Containers & Workflows working group is an informal, multi-vendor working group focused on standards for exchanging Docker-based tools and CWL/WDL workflows, execution of Docker-based tools and workflows on clouds, and abstract access to cloud object stores. +interoperability. The Cloud Work Stream is an informal, multi-vendor working group focused on standards for exchanging Docker-based tools and CWL/WDL workflows, execution of Docker-based tools and workflows on clouds, and abstract access to cloud object stores. What is WES? ============ @@ -26,19 +26,19 @@ UIs that would (possibly) use this API under the hood to facilitate workflow exe Having this standard API supported by multiple execution engines will give people options of processing the same workflow (CWL or WDL) across different workflow execution platforms running across various clouds/environments. As an example use case, one can find a workflow in CWL on [Dockstore.org](http://dockstore.org), use Dockstore to -generate a JSON parameterization file, and submit this a GA4GH-compliant +generate a JSON parameterization file, and submit this to a GA4GH-compliant workflow execution service. Key features of the current API proposal: * ability to request a workflow run using CWL or WDL (and maybe future formats) * ability to parameterize that workflow using a JSON schema (ideally a future version would be in common between CWL and WDL) -* ability to get information about running workflows, status, errors, output file locations etc +* ability to get information about running workflows, status, errors, output file locations, etc. * to search for workflows by arbitrary key/values Outstanding questions: -* a common JSON parameterization format, see work by Peter, is that checked in? +* a common JSON parameterization format * standardizing terms, job, workflow, steps, tools, etc * reference implementation at https://github.com/common-workflow-language/cwltool-service/tree/ga4gh-wes * validation service for testing WES implementations' conformance to the spec @@ -60,8 +60,7 @@ Make sure you have Docker installed for your platform and the `cwltool`. virtualenv env source env/bin/activate - pip install setuptools==28.8.0 - pip install cwl-runner cwltool==1.0.20161114152756 schema-salad==1.18.20161005190847 avro==1.8.1 + pip install -r requirements.txt You can generate the [Swagger](http://swagger.io/) from the Protocol Buffers: @@ -86,4 +85,4 @@ More Information ---------------- * [Global Alliance for Genomics and Health](http://genomicsandhealth.org) -* [Google Forum](https://groups.google.com/forum/#!forum/ga4gh-dwg-containers-workflows) +* [GA4GH Cloud Work Stream)[https://ga4gh.cloud] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..6aa457a8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +cwl-runner +cwltool==1.0.20161114152756 +schema-salad==1.18.20161005190847 +avro==1.8.1 diff --git a/swagger/workflow_execution_service.swagger.json b/swagger/workflow_execution_service.swagger.json index 858cad4a..a1ea37ce 100644 --- a/swagger/workflow_execution_service.swagger.json +++ b/swagger/workflow_execution_service.swagger.json @@ -354,8 +354,8 @@ "title": "OPTIONAL\nThe workflow CWL or WDL document, must provide either this or workflow_url" }, "workflow_params": { - "type": "string", - "title": "REQUIRED\nThe workflow parameterization document (typically a JSON string), includes all parameterizations for\nthe workflow including input and output file locations" + "$ref": "#/definitions/protobufStruct", + "title": "REQUIRED\nThe workflow parameterization document (typically a JSON file), includes all parameterizations for the workflow including input and output file locations" }, "workflow_type": { "type": "string", From 8497809735414f87822e1c27bc74037db77b4878 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Fri, 2 Feb 2018 14:49:50 -0800 Subject: [PATCH 12/15] Change enginer_versions for clarity thanks @briandoconnor --- proto/workflow_execution_service.proto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/proto/workflow_execution_service.proto b/proto/workflow_execution_service.proto index 09ad5636..284cd9dd 100644 --- a/proto/workflow_execution_service.proto +++ b/proto/workflow_execution_service.proto @@ -145,10 +145,10 @@ message ServiceInfo { // supported by this service. repeated string supported_filesystem_protocols = 3; - //The engine(s) used by this WES service, key is engine name e.g. Cromwell and value is version - map engine_versions = 4; + // The engine(s) used by this WES service, key is engine name e.g. Cromwell and value is version + map workflow_engine_versions = 4; - //The system statistics, key is the statistic, value is the count of workflows in that state. + // The system statistics, key is the statistic, value is the count of workflows in that state. // See the State enum for the possible keys. map system_state_counts = 5; From 312f2b24f8e18cdf6dca4c3c7f4b68db94084204 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Fri, 2 Feb 2018 14:53:05 -0800 Subject: [PATCH 13/15] Add auth instructions URL thanks @briandoconnor! --- proto/workflow_execution_service.proto | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/proto/workflow_execution_service.proto b/proto/workflow_execution_service.proto index 284cd9dd..54f4a0cd 100644 --- a/proto/workflow_execution_service.proto +++ b/proto/workflow_execution_service.proto @@ -152,9 +152,13 @@ message ServiceInfo { // See the State enum for the possible keys. map system_state_counts = 5; + // A URL that will help a in generating the tokens necessary to run a workflow using this + // service. + string auth_instructions_url = 6; + // A key-value map of arbitrary, extended metadata outside the scope of the above but useful // to report back - map tags = 6; + map tags = 7; } // Small description of workflows, returned by server during listing From 5c8ab3af85d9f78ea61fca91bcc2b2c025da953b Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Fri, 2 Feb 2018 15:29:05 -0800 Subject: [PATCH 14/15] Add a field for pass additional workflow engine parameters The ServiceInfo endpoint returns default values for each settable parameter in WorkflowRequest. --- proto/workflow_execution_service.proto | 31 +++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/proto/workflow_execution_service.proto b/proto/workflow_execution_service.proto index 54f4a0cd..35980c1a 100644 --- a/proto/workflow_execution_service.proto +++ b/proto/workflow_execution_service.proto @@ -78,7 +78,8 @@ message WorkflowRequest { string workflow_descriptor = 1; // REQUIRED - // The workflow parameterization document (typically a JSON file), includes all parameterizations for the workflow including input and output file locations + // The workflow parameterization document (typically a JSON file), includes all parameterizations for the workflow + // including input and output file locations. google.protobuf.Struct workflow_params = 2; // REQUIRED @@ -93,9 +94,14 @@ message WorkflowRequest { // A key-value map of arbitrary metadata outside the scope of the workflow_params but useful to track with this workflow request map tags = 5; + // OPTIONAL + // Additional parameters can be sent to the workflow engine using this field. Default values + // for these parameters are provided at the ServiceInfo endpoint. + map workflow_engine_parameters = 6; + // OPTIONAL // The workflow CWL or WDL document, must provide either this or workflow_descriptor - string workflow_url = 6; + string workflow_url = 7; } // Request listing of jobs tracked by server @@ -148,17 +154,22 @@ message ServiceInfo { // The engine(s) used by this WES service, key is engine name e.g. Cromwell and value is version map workflow_engine_versions = 4; + // Each workflow engine can present additional parameters that can be sent to the + // workflow engine. This message will list the default values, and their types for each + // workflow engine. + repeated DefaultWorkflowEngineParameter default_workflow_engine_parameters = 5; + // The system statistics, key is the statistic, value is the count of workflows in that state. // See the State enum for the possible keys. - map system_state_counts = 5; + map system_state_counts = 6; // A URL that will help a in generating the tokens necessary to run a workflow using this // service. - string auth_instructions_url = 6; + string auth_instructions_url = 7; // A key-value map of arbitrary, extended metadata outside the scope of the above but useful // to report back - map tags = 7; + map tags = 8; } // Small description of workflows, returned by server during listing @@ -274,4 +285,14 @@ enum State { // The task was canceled by the user. CANCELED = 8; +} + +// A message that allows one to describe default parameters for a workflow +// engine. +message DefaultWorkflowEngineParameter { + // Describes the type of the parameter, e.g. float. + string type = 1; + + // The stringified version of the default parameter. e.g. "2.45". + string default_value = 2; } \ No newline at end of file From f14a2a982d97f38d6054e9c6a08d3ac71212aeb6 Mon Sep 17 00:00:00 2001 From: David Steinberg Date: Fri, 2 Feb 2018 15:48:28 -0800 Subject: [PATCH 15/15] Add comments for using base64 encoding technique. --- proto/workflow_execution_service.proto | 15 +++++-- .../workflow_execution_service.swagger.json | 42 ++++++++++++++++--- 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/proto/workflow_execution_service.proto b/proto/workflow_execution_service.proto index 35980c1a..d02ff933 100644 --- a/proto/workflow_execution_service.proto +++ b/proto/workflow_execution_service.proto @@ -74,7 +74,11 @@ service WorkflowExecutionService { // and executing a given workflow. message WorkflowRequest { // OPTIONAL - // The workflow CWL or WDL document, must provide either this or workflow_url + // The workflow CWL or WDL document, must provide either this or workflow_url. By combining + // this message with a workflow_type_version offered in ServiceInfo, one can initialize + // CWL, WDL, or a base64 encoded gzip of the required workflow descriptors. When files must be + // created in this way, the `workflow_url` should be set to the path of the main + // workflow descriptor. string workflow_descriptor = 1; // REQUIRED @@ -100,7 +104,9 @@ message WorkflowRequest { map workflow_engine_parameters = 6; // OPTIONAL - // The workflow CWL or WDL document, must provide either this or workflow_descriptor + // The workflow CWL or WDL document, must provide either this or workflow_descriptor. When a base64 encoded gzip of + // workflow descriptor files is offered, the `workflow_url` should be set to the relative path + // of the main workflow descriptor. string workflow_url = 7; } @@ -183,7 +189,10 @@ message WorkflowDesc { // Available workflow types supported by a given instance of the service. message WorkflowTypeVersion { - // an array of one or more version strings + // an array of one or more acceptable types for the Workflow Type. For + // example, to send a base64 encoded WDL gzip, one could would offer + // "base64_wdl1.0_gzip". By setting this value, and the path of the main WDL + // to be executed in the workflow_url to "main.wdl" in the WorkflowRequest. repeated string workflow_type_version = 1; } diff --git a/swagger/workflow_execution_service.swagger.json b/swagger/workflow_execution_service.swagger.json index a1ea37ce..d158ae52 100644 --- a/swagger/workflow_execution_service.swagger.json +++ b/swagger/workflow_execution_service.swagger.json @@ -179,6 +179,20 @@ } }, "definitions": { + "ga4gh_wes_DefaultWorkflowEngineParameter": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Describes the type of the parameter, e.g. float." + }, + "default_value": { + "type": "string", + "description": "The stringified version of the default parameter. e.g. \"2.45\"." + } + }, + "description": "A message that allows one to describe default parameters for a workflow\nengine." + }, "ga4gh_wes_Log": { "type": "object", "properties": { @@ -241,13 +255,20 @@ }, "description": "The filesystem protocols supported by this service, currently these may include common\nprotocols such as 'http', 'https', 'sftp', 's3', 'gs', 'file', 'synapse', or others as\nsupported by this service." }, - "engine_versions": { + "workflow_engine_versions": { "type": "object", "additionalProperties": { "type": "string" }, "title": "The engine(s) used by this WES service, key is engine name e.g. Cromwell and value is version" }, + "default_workflow_engine_parameters": { + "type": "array", + "items": { + "$ref": "#/definitions/ga4gh_wes_DefaultWorkflowEngineParameter" + }, + "description": "Each workflow engine can present additional parameters that can be sent to the\nworkflow engine. This message will list the default values, and their types for each\nworkflow engine." + }, "system_state_counts": { "type": "object", "additionalProperties": { @@ -256,6 +277,10 @@ }, "description": "The system statistics, key is the statistic, value is the count of workflows in that state.\nSee the State enum for the possible keys." }, + "auth_instructions_url": { + "type": "string", + "description": "A URL that will help a in generating the tokens necessary to run a workflow using this\nservice." + }, "tags": { "type": "object", "additionalProperties": { @@ -351,11 +376,11 @@ "properties": { "workflow_descriptor": { "type": "string", - "title": "OPTIONAL\nThe workflow CWL or WDL document, must provide either this or workflow_url" + "description": "OPTIONAL\nThe workflow CWL or WDL document, must provide either this or workflow_url. By combining\nthis message with a workflow_type_version offered in ServiceInfo, one can initialize\nCWL, WDL, or a base64 encoded gzip of the required workflow descriptors. When files must be\ncreated in this way, the `workflow_url` should be set to the path of the main\nworkflow descriptor." }, "workflow_params": { "$ref": "#/definitions/protobufStruct", - "title": "REQUIRED\nThe workflow parameterization document (typically a JSON file), includes all parameterizations for the workflow including input and output file locations" + "description": "REQUIRED\nThe workflow parameterization document (typically a JSON file), includes all parameterizations for the workflow\nincluding input and output file locations." }, "workflow_type": { "type": "string", @@ -372,9 +397,16 @@ }, "title": "OPTIONAL\nA key-value map of arbitrary metadata outside the scope of the workflow_params but useful to track with this workflow request" }, + "workflow_engine_parameters": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "OPTIONAL\nAdditional parameters can be sent to the workflow engine using this field. Default values\nfor these parameters are provided at the ServiceInfo endpoint." + }, "workflow_url": { "type": "string", - "title": "OPTIONAL\nThe workflow CWL or WDL document, must provide either this or workflow_descriptor" + "description": "OPTIONAL\nThe workflow CWL or WDL document, must provide either this or workflow_descriptor. When a base64 encoded gzip of\nworkflow descriptor files is offered, the `workflow_url` should be set to the relative path\nof the main workflow descriptor." } }, "description": "To execute a workflow, send a workflow request including all the details needed to begin downloading\nand executing a given workflow." @@ -409,7 +441,7 @@ "items": { "type": "string" }, - "title": "an array of one or more version strings" + "description": "an array of one or more acceptable types for the Workflow Type. For\nexample, to send a base64 encoded WDL gzip, one could would offer\n\"base64_wdl1.0_gzip\". By setting this value, and the path of the main WDL\nto be executed in the workflow_url to \"main.wdl\" in the WorkflowRequest." } }, "description": "Available workflow types supported by a given instance of the service."