From 2d9e36aed2833d3c06f24ab1d6ff97135671af09 Mon Sep 17 00:00:00 2001 From: Gregory-Pereira Date: Wed, 20 Apr 2022 10:57:59 -0700 Subject: [PATCH] creating cron-workflow to invoke job template for the sync-job --- sync-job/base/cron-workflow.yaml | 45 ++++++++ sync-job/base/kustomization.yaml | 11 ++ sync-job/base/workflow-template.yaml | 103 ++++++++++++++++++ sync-job/overlays/common/imagestreamtag.yaml | 11 ++ sync-job/overlays/common/kustomization.yaml | 5 + .../overlays/prod-to-stage/kustomization.yaml | 32 ++++++ .../overlays/stage-to-prod/kustomization.yaml | 32 ++++++ 7 files changed, 239 insertions(+) create mode 100644 sync-job/base/cron-workflow.yaml create mode 100644 sync-job/base/kustomization.yaml create mode 100644 sync-job/base/workflow-template.yaml create mode 100644 sync-job/overlays/common/imagestreamtag.yaml create mode 100644 sync-job/overlays/common/kustomization.yaml create mode 100644 sync-job/overlays/prod-to-stage/kustomization.yaml create mode 100644 sync-job/overlays/stage-to-prod/kustomization.yaml diff --git a/sync-job/base/cron-workflow.yaml b/sync-job/base/cron-workflow.yaml new file mode 100644 index 000000000..6eaad16bb --- /dev/null +++ b/sync-job/base/cron-workflow.yaml @@ -0,0 +1,45 @@ +apiVersion: argoproj.io/v1alpha1 +kind: CronWorkflow +metadata: + name: sync-job-cw + labels: + app: thoth + component: sync +spec: + schedule: "0 10 14 * *" + timezone: "Etc/UTC" + concurrencyPolicy: "Replace" + startingDeadlineSeconds: 0 + workflowSpec: + entrypoint: DocumentSyncTypeOne + templates: + - name: DocumentSyncTypeOne + steps: + - - name: SyncDocuments + templateRef: + name: sync-job-wt + template: SyncDocumentsByType + arguments: + parameters: + - name: THOTH_DOCUMENT_TYPE + value: "solver" + - name: DocumentSyncTypeTwo + steps: + - - name: SyncDocuments + templateRef: + name: sync-job-wt + template: SyncDocumentsByType + arguments: + parameters: + - name: THOTH_DOCUMENT_TYPE + value: "revsolver" + - name: DocumentSyncTypeThree + steps: + - - name: SyncDocuments + templateRef: + name: sync-job-wt + template: SyncDocumentsByType + arguments: + parameters: + - name: THOTH_DOCUMENT_TYPE + value: "security-indicator" diff --git a/sync-job/base/kustomization.yaml b/sync-job/base/kustomization.yaml new file mode 100644 index 000000000..28b72f003 --- /dev/null +++ b/sync-job/base/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +commonLabels: + app.kubernetes.io/name: thoth + app.kubernetes.io/component: sync-job + app.kubernetes.io/managed-by: aicoe-thoth-devops +generatorOptions: + disableNameSuffixHash: true +resources: + - workflow-template.yaml + - cron-workflow.yaml diff --git a/sync-job/base/workflow-template.yaml b/sync-job/base/workflow-template.yaml new file mode 100644 index 000000000..122f583ae --- /dev/null +++ b/sync-job/base/workflow-template.yaml @@ -0,0 +1,103 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: sync-job-wt +spec: + serviceAccountName: pipeline + templates: + - name: SyncDocumentsByType + inputs: + parameters: + - name: THOTH_DOCUMENT_TYPE + description: The type of document to be synced that the workflowTemplate will be invoked with. + enum: + - "adviser" + - "solver" + - "revsolver" + - "package-extract" + - "provenance-checker" + - "security-indicator" + container: + image: "quay.io/thoth-station/sync-job:v0.2.4" + env: + - name: "THOTH_SYNC_FORCE" + value: "0" + - name: "THOTH_SYNC_GRACEFUL" + value: "0" + - name: THOTH_SYNC_DEBUG + value: "0" + - name: THOTH_LOG_SYNC + value: "INFO" + - name: THOTH_DOCUMENT_TYPE + value: "{{inputs.parameters.THOTH_DOCUMENT_TYPE}}" + - name: THOTH_DEPLOYMENT_NAME + valueFrom: + configMapKeyRef: + key: deployment-name + name: thoth + - name: THOTH_S3_ENDPOINT_URL + valueFrom: + configMapKeyRef: + key: host + name: ceph + - name: THOTH_CEPH_BUCKET + valueFrom: + configMapKeyRef: + key: bucket-name + name: ceph + - name: THOTH_CEPH_BUCKET_PREFIX + valueFrom: + configMapKeyRef: + key: bucket-prefix + name: ceph + - name: THOTH_DEPLOYMENT_NAME + valueFrom: + configMapKeyRef: + key: storage-bucket-name + name: thoth + - name: THOTH_CEPH_KEY_ID + valueFrom: + secretKeyRef: + name: ceph + key: key-id + - name: THOTH_CEPH_SECRET_KEY + valueFrom: + secretKeyRef: + name: ceph + key: secret-key + - name: SENTRY_DSN + valueFrom: + secretKeyRef: + name: thoth + key: sentry-dsn + - name: KNOWLEDGE_GRAPH_HOST + valueFrom: + configMapKeyRef: + key: postgresql-host + name: thoth + - name: KNOWLEDGE_GRAPH_PORT + value: "5432" + - name: KNOWLEDGE_GRAPH_SSL_DISABLED + value: "1" + - name: KNOWLEDGE_GRAPH_USER + valueFrom: + secretKeyRef: + name: postgresql + key: database-user + - name: KNOWLEDGE_GRAPH_PASSWORD + valueFrom: + secretKeyRef: + name: postgresql + key: database-password + - name: KNOWLEDGE_GRAPH_DATABASE + valueFrom: + secretKeyRef: + name: postgresql + key: database-name + resources: + limits: + memory: "2Gi" + cpu: 4 + requests: + memory: "2Gi" + cpu: 4 diff --git a/sync-job/overlays/common/imagestreamtag.yaml b/sync-job/overlays/common/imagestreamtag.yaml new file mode 100644 index 000000000..2bc41e238 --- /dev/null +++ b/sync-job/overlays/common/imagestreamtag.yaml @@ -0,0 +1,11 @@ +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: sync-job +spec: + tags: + - name: latest + from: + kind: DockerImage + name: quay.io/thoth-station/sync-job:v0.2.4 + importPolicy: {} diff --git a/sync-job/overlays/common/kustomization.yaml b/sync-job/overlays/common/kustomization.yaml new file mode 100644 index 000000000..8df236d3f --- /dev/null +++ b/sync-job/overlays/common/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - imagestreamtag.yaml + - ../../base diff --git a/sync-job/overlays/prod-to-stage/kustomization.yaml b/sync-job/overlays/prod-to-stage/kustomization.yaml new file mode 100644 index 000000000..dda6927d6 --- /dev/null +++ b/sync-job/overlays/prod-to-stage/kustomization.yaml @@ -0,0 +1,32 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +commonLabels: + app.kubernetes.io/name: thoth + app.kubernetes.io/component: sync-job +resources: + - ../common +patches: + - patch: |- + - op: replace + path: spec/spec/workflowSpec/templates/0/steps/0/0/arguments/parameters/0/value + value: "adviser" + target: + version: v1alpha1 + kind: CronWorkflow + name: sync-job-cw + - patch: |- + - op: replace + path: spec/spec/workflowSpec/templates/1/steps/0/0/arguments/parameters/0/value + value: "package-extract" + target: + version: v1alpha1 + kind: CronWorkflow + name: sync-job-cw + - patch: |- + - op: replace + path: spec/spec/workflowSpec/templates/2/steps/0/0/arguments/parameters/0/value + value: "provenance-checker" + target: + version: v1alpha1 + kind: CronWorkflow + name: sync-job-cw diff --git a/sync-job/overlays/stage-to-prod/kustomization.yaml b/sync-job/overlays/stage-to-prod/kustomization.yaml new file mode 100644 index 000000000..26703e011 --- /dev/null +++ b/sync-job/overlays/stage-to-prod/kustomization.yaml @@ -0,0 +1,32 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +commonLabels: + app.kubernetes.io/name: thoth + app.kubernetes.io/component: sync-job +resources: + - ../common +patches: + - patch: |- + - op: replace + path: spec/spec/workflowSpec/templates/0/steps/0/0/arguments/parameters/0/value + value: "solver" + target: + version: v1alpha1 + kind: CronWorkflow + name: sync-job-cw + - patch: |- + - op: replace + path: spec/spec/workflowSpec/templates/1/steps/0/0/arguments/parameters/0/value + value: "revsolver" + target: + version: v1alpha1 + kind: CronWorkflow + name: sync-job-cw + - patch: |- + - op: replace + path: spec/spec/workflowSpec/templates/2/steps/0/0/arguments/parameters/0/value + value: "security-indicator" + target: + version: v1alpha1 + kind: CronWorkflow + name: sync-job-cw