🌈 Download API for artifacts !

controller/src/argo/argo-client.ts

@@ -3,7 +3,7 @@ import { createInterface } from 'node:readline';
 import type { Got } from 'got';
 /// <reference path="./argo-schema.d.ts" />
 
-export type ArgoTemplate = WorkflowsArgoprojIo.WorkflowsJson.Definitions
+/* export */ type ArgoTemplate = WorkflowsArgoprojIo.WorkflowsJson.Definitions
   .IoArgoprojWorkflowV1alpha1Template;
 
 export type ArgoWorkflow = WorkflowsArgoprojIo.WorkflowsJson.Definitions

controller/src/argo/dry-runs.ts

@@ -4,7 +4,7 @@ import {
 import { DryRunNodeType, DryRunPhase } from '../server/schema.js';
 import getPodName from './get-pod-name.js';
 import type {
-  DryRun, DryRunLogEntry, DryRunNode, DryRunNodePod,
+  DryRun, DryRunLogEntry, DryRunNode, DryRunNodeArtifact, DryRunNodePod,
 } from '../server/schema.js';
 import type { ArgoClientActionNames, ArgoNode, ArgoWorkflow } from './argo-client.js';
 import type ArgoWorkflowClient from './argo-client.js';
@@ -83,8 +83,9 @@ function convertNodeDuration(node: ArgoNode): number | undefined {
     - new Date(node.startedAt).getTime()) / 1000);
 }
 
-export type InternalExtendedDryRunNode = DryRunNode & DryRunNodePod & {
+type InternalExtendedDryRunNode = DryRunNode & Omit<DryRunNodePod, 'podName' | 'metrics' | 'resourcesDuration'> & {
   workflow: ArgoWorkflow;
+  podName: string | undefined;
 };
 
 export function assertDryRunNodeHasWorkflow(
@@ -100,9 +101,19 @@ export function convertArgoWorkflowNode(node: ArgoNode, argoWorkflow: ArgoWorkfl
   const type = convertArgoNodeType(node.type);
 
   let podName: string | undefined;
+  let inputArtifacts: DryRunNodeArtifact[] | undefined;
+  let outputArtifacts: DryRunNodeArtifact[] | undefined;
 
   if (type === DryRunNodeType.Pod) {
     podName = getPodName(node, argoWorkflow);
+    inputArtifacts = node.inputs?.artifacts?.map(({ name, s3 }) => ({
+      name,
+      key: s3?.key,
+    }));
+    outputArtifacts = node.outputs?.artifacts?.map(({ name, s3 }) => ({
+      name,
+      key: s3?.key,
+    }));
   }
 
   return {
@@ -121,6 +132,8 @@ export function convertArgoWorkflowNode(node: ArgoNode, argoWorkflow: ArgoWorkfl
     duration: convertNodeDuration(node),
     workflow: argoWorkflow,
     podName,
+    inputArtifacts,
+    outputArtifacts,
   };
 }
 

controller/src/config.ts

@@ -18,7 +18,7 @@ export const minioEndpoint = process.env.MINIO_ENDPOINT ?? 'localhost';
 export const minioPort = process.env.MINIO_PORT
   ? Number.parseInt(process.env.MINIO_PORT, 10) : undefined;
 export const minioUseSSL = !!process.env.MINIO_USE_SSL;
-export const minioBucketName = process.env.MINIO_BUCKET_NAME ?? 'simpipe2';
+export const minioBucketName = process.env.MINIO_BUCKET_NAME ?? 'artifacts';
 export const minioRegion = process.env.MINIO_REGION ?? 'no-region';
 
 // Argo client

controller/src/server/resolvers.ts

@@ -18,7 +18,7 @@ import {
 import {
   createProject, deleteProject, getProject, projects, renameProject,
 } from '../k8s/projects.js';
-import { computePresignedPutUrl } from '../minio/minio.js';
+import { computePresignedGetUrl, computePresignedPutUrl } from '../minio/minio.js';
 import queryPrometheusResolver from '../prometheus/query-prometheus-resolver.js';
 import { PingError } from './apollo-errors.js';
 import type { ArgoWorkflow } from '../argo/argo-client.js';
@@ -29,11 +29,13 @@ import type {
   DryRunLogArgs as DryRunLogArguments,
   DryRunNode,
   DryRunNodeArgs as DryRunNodeArguments,
+  DryRunNodeArtifact,
   DryRunNodeMetrics,
   DryRunNodeMetricsCpuSystemSecondsTotalArgs as DryRunNodeMetricsCpuSystemSecondsTotalArguments,
   DryRunNodePod, DryRunNodePodLogArgs as DryRunNodePodLogArguments,
   Mutation,
   MutationAssignDryRunToProjectArgs as MutationAssignDryRunToProjectArguments,
+  MutationComputeUploadPresignedUrlArgs as MutationComputeUploadPresignedUrlArguments,
   MutationCreateDockerRegistryCredentialArgs as MutationCreateDockerRegistryCredentialArguments,
   MutationCreateDryRunArgs as MutationCreateDryRunArguments,
   MutationCreateProjectArgs as MutationCreateProjectArguments,
@@ -104,19 +106,6 @@ const resolvers = {
 
       return 'pong';
     },
-    async computeUploadPresignedUrl(
-      _p: EmptyParent, _a: EmptyArguments, context: AuthenticatedContext,
-    ): Promise<Query['computeUploadPresignedUrl']> {
-      const { sub } = context.user;
-      // Make sure the user is a filesystem safe string
-      if (!/^[\w-]+$/i.test(sub)) {
-        throw new Error('User identifier (sub) is unsupported for files');
-      }
-      const uuid = randomUUID();
-      const objectName = `${sub}/${uuid}`;
-      const url = await computePresignedPutUrl(objectName);
-      return url;
-    },
     async dockerRegistryCredentials(
       _p: EmptyParent, _a: EmptyArguments, context: AuthenticatedContext,
     ): Promise<Query['dockerRegistryCredentials']> {
@@ -288,6 +277,29 @@ const resolvers = {
       const { k8sClient, k8sNamespace } = context;
       return await deleteProject(projectId, k8sClient, k8sNamespace);
     },
+    async computeUploadPresignedUrl(
+      _p: EmptyParent,
+      _arguments: MutationComputeUploadPresignedUrlArguments,
+      context: AuthenticatedContext,
+    ): Promise<Mutation['computeUploadPresignedUrl']> {
+      const { sub } = context.user;
+      // Make sure the user is a filesystem safe string
+      if (!/^[\w-]+$/i.test(sub)) {
+        throw new Error('User identifier (sub) is unsupported for files');
+      }
+
+      let { key } = _arguments;
+      if (key) {
+        if (!/^[\w.-]+$/i.test(key)) {
+          throw new Error('Key is unsupported for files');
+        }
+      } else {
+        key = randomUUID();
+      }
+
+      const objectName = `${sub}/${key}`;
+      return await computePresignedPutUrl(objectName);
+    },
   } as Required<MutationResolvers<AuthenticatedContext, EmptyParent>>,
   Project: {
     async dryRuns(
@@ -442,6 +454,17 @@ const resolvers = {
     threadsMax: queryPrometheusResolver.bind(undefined, 'simpipe_threads_max'),
     ulimitsSoft: queryPrometheusResolver.bind(undefined, 'simpipe_ulimits_soft'),
   },
+  DryRunNodeArtifact: {
+    async url(
+      dryRunNodeArtifact: DryRunNodeArtifact,
+    ): Promise<DryRunNodeArtifact['url']> {
+      const { key } = dryRunNodeArtifact;
+      if (!key) {
+        return undefined;
+      }
+      return await computePresignedGetUrl(key);
+    },
+  },
 };
 
 export default resolvers;

controller/src/server/schema.graphql

@@ -13,8 +13,6 @@ scalar PrometheusStringNumber
 type Query {
   """ Fetch the current username. """
   username: String! @auth
-  """ Compute a presigned URL for uploading a file using a HTTP PUT. """
-  computeUploadPresignedUrl: String! @auth
   """ Returns pong if the server is up and running. """
   ping: String!
   """ List of docker registry credentials. """
@@ -198,6 +196,12 @@ type DryRunNodePod implements DryRunNode {
 
   """ The logs of the node. """
   log(maxLines: Int, grep: String): [String!]
+
+  """ Input artifacts of the node. """
+  inputArtifacts: [DryRunNodeArtifact!]
+
+  """ Output artifacts of the node. """
+  outputArtifacts: [DryRunNodeArtifact!]
 }
 
 type DryRunNodeMisc implements DryRunNode {
@@ -265,6 +269,15 @@ type DryRunNodeMetrics {
   ulimitsSoft(start: TimeStamp, end: TimeStamp, step: Int): [PrometheusSample!]
 }
 
+type DryRunNodeArtifact {
+  """ The artifact name """
+  name: String!
+  """ The artifact path """
+  key: String
+  """ URL to download the artifact using an HTTP GET."""
+  url: String
+}
+
 """
   Resources durations. Estimation from Argo, usually pretty not accurate.
   See https://argoproj.github.io/argo-workflows/resource-duration/
@@ -369,4 +382,7 @@ type Mutation @auth {
 
   """ Update a docker registry credential """
   updateDockerRegistryCredential(credential: DockerRegistryCredentialInput!): DockerRegistryCredential!
+
+  """ Compute a presigned URL for uploading a file using a HTTP PUT. """
+  computeUploadPresignedUrl(key: String): String!
 }

controller/src/server/schema.ts

@@ -135,6 +135,16 @@ export type DryRunNode = {
   type: DryRunNodeType;
 };
 
+export type DryRunNodeArtifact = {
+  __typename?: 'DryRunNodeArtifact';
+  /**  The artifact path  */
+  key?: Maybe<Scalars['String']['output']>;
+  /**  The artifact name  */
+  name: Scalars['String']['output'];
+  /**  URL to download the artifact using an HTTP GET. */
+  url?: Maybe<Scalars['String']['output']>;
+};
+
 /**  Prometheus metrics for the node.  */
 export type DryRunNodeMetrics = {
   __typename?: 'DryRunNodeMetrics';
@@ -533,11 +543,15 @@ export type DryRunNodePod = DryRunNode & {
   exitCode?: Maybe<Scalars['String']['output']>;
   finishedAt?: Maybe<Scalars['String']['output']>;
   id: Scalars['String']['output'];
+  /**  Input artifacts of the node.  */
+  inputArtifacts?: Maybe<Array<DryRunNodeArtifact>>;
   /**  The logs of the node.  */
   log?: Maybe<Array<Scalars['String']['output']>>;
   /**  The name of the pod.  */
   metrics: DryRunNodeMetrics;
   name: Scalars['String']['output'];
+  /**  Output artifacts of the node.  */
+  outputArtifacts?: Maybe<Array<DryRunNodeArtifact>>;
   phase: DryRunPhase;
   /**  The name of the pod.  */
   podName: Scalars['String']['output'];
@@ -625,6 +639,8 @@ export type Mutation = {
   __typename?: 'Mutation';
   /**  Assign a dry run to a project  */
   assignDryRunToProject: DryRun;
+  /**  Compute a presigned URL for uploading a file using a HTTP PUT.  */
+  computeUploadPresignedUrl: Scalars['String']['output'];
   /**  Create a docker registry credential  */
   createDockerRegistryCredential: DockerRegistryCredential;
   /**
@@ -666,6 +682,11 @@ export type MutationAssignDryRunToProjectArgs = {
 };
 
 
+export type MutationComputeUploadPresignedUrlArgs = {
+  key?: InputMaybe<Scalars['String']['input']>;
+};
+
+
 export type MutationCreateDockerRegistryCredentialArgs = {
   credential: DockerRegistryCredentialInput;
 };
@@ -754,8 +775,6 @@ export type PrometheusSample = {
 
 export type Query = {
   __typename?: 'Query';
-  /**  Compute a presigned URL for uploading a file using a HTTP PUT.  */
-  computeUploadPresignedUrl: Scalars['String']['output'];
   /**  List of docker registry credentials.  */
   dockerRegistryCredentials: Array<DockerRegistryCredential>;
   /**  Get a dry run by id  */
@@ -864,6 +883,7 @@ export type ResolversTypes = {
   DryRun: ResolverTypeWrapper<DryRun>;
   DryRunLogEntry: ResolverTypeWrapper<DryRunLogEntry>;
   DryRunNode: ResolverTypeWrapper<ResolversInterfaceTypes<ResolversTypes>['DryRunNode']>;
+  DryRunNodeArtifact: ResolverTypeWrapper<DryRunNodeArtifact>;
   DryRunNodeMetrics: ResolverTypeWrapper<DryRunNodeMetrics>;
   DryRunNodeMisc: ResolverTypeWrapper<DryRunNodeMisc>;
   DryRunNodePod: ResolverTypeWrapper<DryRunNodePod>;
@@ -892,6 +912,7 @@ export type ResolversParentTypes = {
   DryRun: DryRun;
   DryRunLogEntry: DryRunLogEntry;
   DryRunNode: ResolversInterfaceTypes<ResolversParentTypes>['DryRunNode'];
+  DryRunNodeArtifact: DryRunNodeArtifact;
   DryRunNodeMetrics: DryRunNodeMetrics;
   DryRunNodeMisc: DryRunNodeMisc;
   DryRunNodePod: DryRunNodePod;
@@ -951,6 +972,13 @@ export type DryRunNodeResolvers<ContextType = any, ParentType extends ResolversP
   type?: Resolver<ResolversTypes['DryRunNodeType'], ParentType, ContextType>;
 };
 
+export type DryRunNodeArtifactResolvers<ContextType = any, ParentType extends ResolversParentTypes['DryRunNodeArtifact'] = ResolversParentTypes['DryRunNodeArtifact']> = {
+  key?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  url?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
 export type DryRunNodeMetricsResolvers<ContextType = any, ParentType extends ResolversParentTypes['DryRunNodeMetrics'] = ResolversParentTypes['DryRunNodeMetrics']> = {
   cpuSystemSecondsTotal?: Resolver<Maybe<Array<ResolversTypes['PrometheusSample']>>, ParentType, ContextType, Partial<DryRunNodeMetricsCpuSystemSecondsTotalArgs>>;
   cpuUsageSecondsTotal?: Resolver<Maybe<Array<ResolversTypes['PrometheusSample']>>, ParentType, ContextType, Partial<DryRunNodeMetricsCpuUsageSecondsTotalArgs>>;
@@ -1018,9 +1046,11 @@ export type DryRunNodePodResolvers<ContextType = any, ParentType extends Resolve
   exitCode?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   finishedAt?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   id?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  inputArtifacts?: Resolver<Maybe<Array<ResolversTypes['DryRunNodeArtifact']>>, ParentType, ContextType>;
   log?: Resolver<Maybe<Array<ResolversTypes['String']>>, ParentType, ContextType, Partial<DryRunNodePodLogArgs>>;
   metrics?: Resolver<ResolversTypes['DryRunNodeMetrics'], ParentType, ContextType>;
   name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  outputArtifacts?: Resolver<Maybe<Array<ResolversTypes['DryRunNodeArtifact']>>, ParentType, ContextType>;
   phase?: Resolver<ResolversTypes['DryRunPhase'], ParentType, ContextType>;
   podName?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   progress?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
@@ -1050,6 +1080,7 @@ export type DryRunStatusResolvers<ContextType = any, ParentType extends Resolver
 
 export type MutationResolvers<ContextType = any, ParentType extends ResolversParentTypes['Mutation'] = ResolversParentTypes['Mutation']> = {
   assignDryRunToProject?: Resolver<ResolversTypes['DryRun'], ParentType, ContextType, RequireFields<MutationAssignDryRunToProjectArgs, 'dryRunId' | 'projectId'>>;
+  computeUploadPresignedUrl?: Resolver<ResolversTypes['String'], ParentType, ContextType, Partial<MutationComputeUploadPresignedUrlArgs>>;
   createDockerRegistryCredential?: Resolver<ResolversTypes['DockerRegistryCredential'], ParentType, ContextType, RequireFields<MutationCreateDockerRegistryCredentialArgs, 'credential'>>;
   createDryRun?: Resolver<ResolversTypes['DryRun'], ParentType, ContextType, RequireFields<MutationCreateDryRunArgs, 'argoWorkflow'>>;
   createProject?: Resolver<ResolversTypes['Project'], ParentType, ContextType, RequireFields<MutationCreateProjectArgs, 'project'>>;
@@ -1084,7 +1115,6 @@ export interface PrometheusStringNumberScalarConfig extends GraphQLScalarTypeCon
 }
 
 export type QueryResolvers<ContextType = any, ParentType extends ResolversParentTypes['Query'] = ResolversParentTypes['Query']> = {
-  computeUploadPresignedUrl?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   dockerRegistryCredentials?: Resolver<Array<ResolversTypes['DockerRegistryCredential']>, ParentType, ContextType>;
   dryRun?: Resolver<ResolversTypes['DryRun'], ParentType, ContextType, RequireFields<QueryDryRunArgs, 'dryRunId'>>;
   ping?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
@@ -1103,6 +1133,7 @@ export type Resolvers<ContextType = any> = {
   DryRun?: DryRunResolvers<ContextType>;
   DryRunLogEntry?: DryRunLogEntryResolvers<ContextType>;
   DryRunNode?: DryRunNodeResolvers<ContextType>;
+  DryRunNodeArtifact?: DryRunNodeArtifactResolvers<ContextType>;
   DryRunNodeMetrics?: DryRunNodeMetricsResolvers<ContextType>;
   DryRunNodeMisc?: DryRunNodeMiscResolvers<ContextType>;
   DryRunNodePod?: DryRunNodePodResolvers<ContextType>;