Add additional shards and attempts data to the Job Details page (#641)

* Added the ability to see all attempts for a task or shard
* Task-level and shard attempt-level inputs and outputs have been added
* Implemented UI for scattered sub-workflows
* Re-implemented shard data requests to get data more efficiently from the API

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Job Manager Change Log
 
+## v1.0.0 Release Notes
+
+### Multiple changes to the "List" tab in the Job Details page for Cromwell jobs.
+
+#### If a task or shard has multiple attempts it can be clicked on, which will expand to show additional rows, one for each attempt.
+
+#### Scattered task names are a link that will display scattered task shards in a dialog box.
+
+#### Scattered sub-workflow shards are a link that will take the users to a "Job Details" page with details of that shard.
+
+#### Added icons for inputs and outputs which can be clicked on to show the full list.
+
+#### Changed the call-cached icon and moved it into the "duration" column.
+
+### Re-implemented shard data requests to get data more efficiently from the API.
+
 ## v0.7.3 Release Notes
 
 ### Fixed the broken display of icons in Firefox.

README.md

@@ -8,8 +8,6 @@
 [![Github](https://img.shields.io/badge/Docker%20Image-dsub%20shim%20API-blue.svg?style=flat-square)](https://cloud.docker.com/u/databiosphere/repository/docker/databiosphere/job-manager-api-dsub)
 [![Github](https://img.shields.io/badge/Docker%20Image-cromwell%20shim%20API-blue.svg?style=flat-square)](https://cloud.docker.com/u/databiosphere/repository/docker/databiosphere/job-manager-api-cromwell)
 
-_This product is in Alpha and not yet ready for production use. We welcome all feedback!_
-
 ## User facing documentation
 
 Welcome to the Job Manager repository! If you're a developer you're in the right place.

api/jobs.yaml

@@ -137,6 +137,71 @@ paths:
           $ref: '#/responses/ServerError'
       tags:
         - Jobs
+  '/jobs/{id}/{task}/attempts':
+    get:
+      operationId: GetTaskAttempts
+      summary: Query for task-level metadata for a specified job
+      parameters:
+        - name: id
+          description: Job ID
+          required: true
+          type: string
+          in: path
+        - name: task
+          description: task name
+          required: true
+          type: string
+          in: path
+      responses:
+        '200':
+          description: Success
+          schema:
+            $ref: '#/definitions/JobAttemptsResponse'
+        '400':
+          $ref: '#/responses/BadRequest'
+        '401':
+          $ref: '#/responses/Unauthorized'
+        '404':
+          $ref: '#/responses/NotFound'
+        '500':
+          $ref: '#/responses/ServerError'
+      tags:
+        - Jobs
+  '/jobs/{id}/{task}/{index}/attempts':
+    get:
+      operationId: GetShardAttempts
+      summary: Query for shard-level metadata for a specified job
+      parameters:
+        - name: id
+          description: Job ID
+          required: true
+          type: string
+          in: path
+        - name: task
+          description: task name
+          required: true
+          type: string
+          in: path
+        - name: index
+          description: shard index
+          required: true
+          type: string
+          in: path
+      responses:
+        '200':
+          description: Success
+          schema:
+            $ref: '#/definitions/JobAttemptsResponse'
+        '400':
+          $ref: '#/responses/BadRequest'
+        '401':
+          $ref: '#/responses/Unauthorized'
+        '404':
+          $ref: '#/responses/NotFound'
+        '500':
+          $ref: '#/responses/ServerError'
+      tags:
+        - Jobs
   '/aggregations':
     get:
       operationId: GetJobAggregations
@@ -194,6 +259,7 @@ definitions:
         description: OAuth 2.0 requested scopes
         items:
           type: string
+
   DisplayField:
     description: Description of a display field
     required:
@@ -230,6 +296,7 @@ definitions:
           this field in a name:value way.
         type: boolean
         default: false
+
   CapabilitiesResponse:
     description: Specification of the backend's capabilities
     properties:
@@ -260,6 +327,7 @@ definitions:
         description: Time frames supported by the /aggregations endpoint.
         items:
           $ref: '#/definitions/TimeFrame'
+
   JobMetadataResponse:
     description: Job and task level metadata
     required:
@@ -302,6 +370,7 @@ definitions:
           $ref: '#/definitions/FailureMessage'
       extensions:
         $ref: '#/definitions/ExtendedFields'
+
   UpdateJobLabelsRequest:
     description: Update labels on a job
     properties:
@@ -311,12 +380,14 @@ definitions:
           Labels to be merged into the existing job labels. Overwrites the label
           values for all given keys. Existing labels on the job which are not
           referenced in this request are left unmodified.
+
   UpdateJobLabelsResponse:
     description: Response to an update labels request
     properties:
       labels:
         type: object
         description: All job labels, after the update has been applied.
+
   AggregationResponse:
     description: Jobs aggregated by label and status within the specified time frame.
     properties:
@@ -326,6 +397,7 @@ definitions:
         type: array
         items:
           $ref: '#/definitions/Aggregation'
+
   Aggregation:
     description: Aggregation of jobs by statuses for a given label, within a specified time frame.
     properties:
@@ -337,6 +409,7 @@ definitions:
         type: array
         items:
           $ref: '#/definitions/AggregationEntry'
+
   AggregationEntry:
     description: An aggregation entry for a given label value.
     properties:
@@ -345,20 +418,23 @@ definitions:
         type: string
       statusCounts:
         $ref: '#/definitions/StatusCounts'
+
   StatusCounts:
     description: A collection of StatusCount
     properties:
       counts:
         type: array
         items:
           $ref: '#/definitions/StatusCount'
+
   StatusCount:
     description: A mapping of JobStatus to number of jobs with that status.
     properties:
       status:
         $ref: '#/definitions/JobStatus'
       count:
         type: integer
+
   TimeFrame:
     type: string
     enum: *TIMEFRAME
@@ -421,6 +497,7 @@ definitions:
           of `nextPageToken` from the previous response.
       extensions:
         $ref: '#/definitions/ExtendedQueryFields'
+
   ExtendedQueryFields:
     description: >
       Extended fields to query by which may not be supported by all backends.
@@ -439,6 +516,7 @@ definitions:
         type: boolean
         description: >
           Returns only jobs that do not have the label 'flag' set to 'archive'
+
   QueryJobsResponse:
     description: Response to a job query
     required:
@@ -461,6 +539,7 @@ definitions:
           sets. Provide this value in a subsequent request to return the next
           page of results. This field will be empty if there aren't any
           additional results.
+
   # TODO(calbach): Reconcile QueryJobsResult with JobMetadata.
   QueryJobsResult:
     description: Result for an individual job returned by a job query
@@ -494,6 +573,7 @@ definitions:
         description: Custom job labels with string values
       extensions:
         $ref: '#/definitions/ExtendedFields'
+
   JobStatus:
     type: string
     enum:
@@ -504,6 +584,7 @@ definitions:
       - Failed
       - Succeeded
       - Aborted
+
   ExtendedFields:
     description: >
       Extended fields on jobs which may not be populated by all backends. See
@@ -555,6 +636,9 @@ definitions:
       inputs:
         type: object
         description: Mapping of input fully qualified names to stringified values
+      outputs:
+        type: object
+        description: Mapping of input fully qualified names to stringified values
       # TODO: Define a generalized enum for this. For details, see
       # https://github.com/DataBiosphere/job-manager/pull/14.
       executionStatus:
@@ -574,11 +658,11 @@ definitions:
       shards:
         type: array
         items:
-          $ref: '#/definitions/TaskShard'
-      failures:
+          $ref: '#/definitions/Shard'
+      failureMessages:
         type: array
         items:
-          $ref: '#/definitions/FailureMessage'
+          type: string
       returnCode:
         type: integer
         description: Task execution return code
@@ -588,12 +672,16 @@ definitions:
       stderr:
         type: string
         description: Path to the standard error file for this task
-      attempts:
-        type: integer
-        description: Number of times the task was run
       callRoot:
         type: string
         description: The root directory of the task execution
+      attempts:
+        type: integer
+        description: Number of times the task was run
+      attemptsData:
+        type: array
+        items:
+            $ref: '#/definitions/IndividualAttempt'
       jobId:
         type: string
         description: >
@@ -609,8 +697,56 @@ definitions:
         items:
           $ref: '#/definitions/ExecutionEvent'
 
-  TaskShard:
-    description: Individual shards for a scattered task
+  IndividualAttempt:
+    description: The most granular collection of data for a job.
+    properties:
+      attemptNumber:
+        type: integer
+      executionStatus:
+        type: string
+        description: Backend-specific task status.
+      callCached:
+        type: boolean
+        description: A flag to indicate if task was call-cached.
+        default: false
+      stdout:
+        type: string
+        description: Path to the standard output file for this task
+      stderr:
+        type: string
+        description: Path to the standard error file for this task
+      callRoot:
+        type: string
+        description: The root directory of the task execution
+      inputs:
+        type: object
+        description: Mapping of input fully qualified names to stringified values
+      outputs:
+        type: object
+        description: Mapping of input fully qualified names to stringified values
+      start:
+        type: string
+        format: date-time
+        description: Start datetime of the task execution in ISO8601 format with milliseconds
+      end:
+        type: string
+        format: date-time
+        description: End datetime of the task execution in ISO8601 format with milliseconds
+      failureMessages:
+        type: array
+        items:
+          type: string
+
+  JobAttemptsResponse:
+    description: Attempts for specified task or shard
+    properties:
+      attempts:
+        type: array
+        items:
+          $ref: '#/definitions/IndividualAttempt'
+
+  Shard:
+    description: Individual shards for a scattered task or sub-workflow
     properties:
       executionStatus:
         type: string
@@ -625,10 +761,34 @@ definitions:
         description: End datetime of the task execution in ISO8601 format with milliseconds
       shardIndex:
         type: integer
+      stdout:
+        type: string
+        description: If from a task, path to the standard output file for this task or shard
+      stderr:
+        type: string
+        description: If from a task, path to the standard error file for this task or shard
+      callRoot:
+        type: string
+        description: If from a task, the root directory of the task or shard execution
+      attempts:
+        type: integer
+        description: Number of times the task was run
+      failureMessages:
+        type: array
+        items:
+          type: string
+      attemptsData:
+        type: array
+        items:
+          $ref: '#/definitions/IndividualAttempt'
       executionEvents:
         type: array
         items:
           $ref: '#/definitions/ExecutionEvent'
+      jobId:
+        type: string
+        description: >
+          Id of the job corresponding to this shard, if this is a scattered sub-workflow shard
 
   FailureMessage:
     description: >