diff --git a/sync-job/base/cron-workflow.yaml b/sync-job/base/cron-workflow.yaml new file mode 100644 index 000000000..08897f088 --- /dev/null +++ b/sync-job/base/cron-workflow.yaml @@ -0,0 +1,269 @@ +apiVersion: argoproj.io/v1alpha1 +kind: CronWorkflow +metadata: + name: sync-job-cw + labels: + app: thoth + component: sync +spec: + schedule: "0 10 14 * *" + timezone: "Etc/UTC" + concurrencyPolicy: "Replace" + startingDeadlineSeconds: 0 + workflowSpec: + entrypoint: SyncDocumentTypeOne + paralellism: 4 + templates: + - name: SyncDocumentTypeOne + container: + image: "quay.io/thoth-station/sync-job:v0.2.3-dev" + command: ["python3", "app.py"] + env: + - name: THOTH_SYNC_FORCE + value: "0" + - name: THOTH_SYNC_GRACEFUL + value: "0" + - name: THOTH_SYNC_DEBUG + value: "0" + - name: THOTH_LOG_SYNC + value: "INFO" + - name: THOTH_DOCUMENT_TYPE + value: "${THOTH_DOCUMENT_TYPE}" + - name: THOTH_DEPLOYMENT_NAME + valueFrom: + configMapKeyRef: + key: deployment-name + name: thoth + - name: THOTH_S3_ENDPOINT_URL + valueFrom: + configMapKeyRef: + key: host + name: ceph + - name: THOTH_CEPH_BUCKET + valueFrom: + configMapKeyRef: + key: bucket-name + name: ceph + - name: THOTH_CEPH_BUCKET_PREFIX + valueFrom: + configMapKeyRef: + key: bucket-prefix + name: ceph + - name: THOTH_DEPLOYMENT_NAME + valueFrom: + configMapKeyRef: + key: storage-bucket-name + name: thoth + - name: THOTH_CEPH_KEY_ID + valueFrom: + secretKeyRef: + name: ceph + key: key-id + - name: THOTH_CEPH_SECRET_KEY + valueFrom: + secretKeyRef: + name: ceph + key: secret-key + - name: SENTRY_DSN + valueFrom: + secretKeyRef: + name: thoth + key: sentry-dsn + - name: KNOWLEDGE_GRAPH_HOST + valueFrom: + configMapKeyRef: + key: postgresql-host + name: thoth + - name: KNOWLEDGE_GRAPH_PORT + value: "5432" + - name: KNOWLEDGE_GRAPH_SSL_DISABLED + value: "1" + - name: KNOWLEDGE_GRAPH_USER + valueFrom: + secretKeyRef: + name: postgresql + key: database-user + - name: KNOWLEDGE_GRAPH_PASSWORD + valueFrom: + secretKeyRef: + name: postgresql + key: database-password + - name: KNOWLEDGE_GRAPH_DATABASE + valueFrom: + secretKeyRef: + name: postgresql + key: database-name + resources: + limits: + memory: "2Gi" + cpu: 4 + requests: + memory: "2Gi" + cpu: 4 + - name: SyncDocumentTypeTwo + container: + image: "quay.io/thoth-station/sync-job:v0.2.3-dev" + command: ["python3", "app.py"] + env: + - name: THOTH_SYNC_FORCE + value: "0" + - name: THOTH_SYNC_GRACEFUL + value: "0" + - name: THOTH_SYNC_DEBUG + value: "0" + - name: THOTH_LOG_SYNC + value: "INFO" + - name: THOTH_DOCUMENT_TYPE + value: "${THOTH_DOCUMENT_TYPE}" + - name: THOTH_DEPLOYMENT_NAME + valueFrom: + configMapKeyRef: + key: deployment-name + name: thoth + - name: THOTH_S3_ENDPOINT_URL + valueFrom: + configMapKeyRef: + key: host + name: ceph + - name: THOTH_CEPH_BUCKET + valueFrom: + configMapKeyRef: + key: bucket-name + name: ceph + - name: THOTH_CEPH_BUCKET_PREFIX + valueFrom: + configMapKeyRef: + key: bucket-prefix + name: ceph + - name: THOTH_CEPH_KEY_ID + valueFrom: + secretKeyRef: + name: ceph + key: key-id + - name: THOTH_CEPH_SECRET_KEY + valueFrom: + secretKeyRef: + name: ceph + key: secret-key + - name: SENTRY_DSN + valueFrom: + secretKeyRef: + name: thoth + key: sentry-dsn + - name: KNOWLEDGE_GRAPH_HOST + valueFrom: + configMapKeyRef: + key: postgresql-host + name: thoth + - name: KNOWLEDGE_GRAPH_PORT + value: "5432" + - name: KNOWLEDGE_GRAPH_SSL_DISABLED + value: "1" + - name: KNOWLEDGE_GRAPH_USER + valueFrom: + secretKeyRef: + name: postgresql + key: database-user + - name: KNOWLEDGE_GRAPH_PASSWORD + valueFrom: + secretKeyRef: + name: postgresql + key: database-password + - name: KNOWLEDGE_GRAPH_DATABASE + valueFrom: + secretKeyRef: + name: postgresql + key: database-name + resources: + limits: + memory: "2Gi" + cpu: 4 + requests: + memory: "2Gi" + cpu: 4 + - name: SyncDocumentTypeThree + container: + image: "quay.io/thoth-station/sync-job:v0.2.3-dev" + command: ["python3", "app.py"] + env: + - name: THOTH_SYNC_FORCE + value: "0" + - name: THOTH_SYNC_GRACEFUL + value: "0" + - name: THOTH_SYNC_DEBUG + value: "0" + - name: THOTH_LOG_SYNC + value: "INFO" + - name: THOTH_DOCUMENT_TYPE + value: "${THOTH_DOCUMENT_TYPE}" + - name: THOTH_DEPLOYMENT_NAME + valueFrom: + configMapKeyRef: + key: deployment-name + name: thoth + - name: THOTH_S3_ENDPOINT_URL + valueFrom: + configMapKeyRef: + key: host + name: ceph + - name: THOTH_CEPH_BUCKET + valueFrom: + configMapKeyRef: + key: bucket-name + name: ceph + - name: THOTH_CEPH_BUCKET_PREFIX + valueFrom: + configMapKeyRef: + key: bucket-prefix + name: ceph + - name: THOTH_DEPLOYMENT_NAME + valueFrom: + configMapKeyRef: + key: storage-bucket-name + name: thoth + - name: THOTH_CEPH_KEY_ID + valueFrom: + secretKeyRef: + name: ceph + key: key-id + - name: THOTH_CEPH_SECRET_KEY + valueFrom: + secretKeyRef: + name: ceph + key: secret-key + - name: SENTRY_DSN + valueFrom: + secretKeyRef: + name: thoth + key: sentry-dsn + - name: KNOWLEDGE_GRAPH_HOST + valueFrom: + configMapKeyRef: + key: postgresql-host + name: thoth + - name: KNOWLEDGE_GRAPH_PORT + value: "5432" + - name: KNOWLEDGE_GRAPH_SSL_DISABLED + value: "1" + - name: KNOWLEDGE_GRAPH_USER + valueFrom: + secretKeyRef: + name: postgresql + key: database-user + - name: KNOWLEDGE_GRAPH_PASSWORD + valueFrom: + secretKeyRef: + name: postgresql + key: database-password + - name: KNOWLEDGE_GRAPH_DATABASE + valueFrom: + secretKeyRef: + name: postgresql + key: database-name + resources: + limits: + memory: "2Gi" + cpu: 4 + requests: + memory: "2Gi" + cpu: 4 diff --git a/sync-job/base/kustomization.yaml b/sync-job/base/kustomization.yaml new file mode 100644 index 000000000..5585fe779 --- /dev/null +++ b/sync-job/base/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +commonLabels: + app.kubernetes.io/name: thoth + app.kubernetes.io/component: sync-job + app.kubernetes.io/managed-by: aicoe-thoth-devops +generatorOptions: + disableNameSuffixHash: true +resources: + - cron-workflow.yaml diff --git a/sync-job/overlays/common/imagestreamtag.yaml b/sync-job/overlays/common/imagestreamtag.yaml new file mode 100644 index 000000000..c6962b092 --- /dev/null +++ b/sync-job/overlays/common/imagestreamtag.yaml @@ -0,0 +1,11 @@ +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: sync-job +spec: + tags: + - name: latest + from: + kind: DockerImage + name: quay.io/thoth-station/sync-job:v0.2.3-dev + importPolicy: {} diff --git a/sync-job/overlays/common/kustomization.yaml b/sync-job/overlays/common/kustomization.yaml new file mode 100644 index 000000000..8df236d3f --- /dev/null +++ b/sync-job/overlays/common/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - imagestreamtag.yaml + - ../../base diff --git a/sync-job/overlays/prod-to-stage/kustomization.yaml b/sync-job/overlays/prod-to-stage/kustomization.yaml new file mode 100644 index 000000000..04636063d --- /dev/null +++ b/sync-job/overlays/prod-to-stage/kustomization.yaml @@ -0,0 +1,32 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +commonLabels: + app.kubernetes.io/name: thoth + app.kubernetes.io/component: sync-job +resources: + - ../common/ +patches: + - patch: |- + - op: add + path: /spec/workflowSpec/templates/container/0/env/5/value + value: "adviser" + target: + version: v1alpha1 + kind: CronWorkflow + name: sync-job-cw + - patch: |- + - op: add + path: /spec/workflowSpec/templates/container/1/env/5/value + value: ["package-extract"] + target: + version: v1alpha1 + kind: CronWorkflow + name: sync-job-cw + - patch: |- + - op: add + path: /spec/workflowSpec/templates/container/2/env/5/value + value: "provenance-checker" + target: + version: v1alpha1 + kind: CronWorkflow + name: sync-job-cw diff --git a/sync-job/overlays/stage-to-prod/kustomization.yaml b/sync-job/overlays/stage-to-prod/kustomization.yaml new file mode 100644 index 000000000..3d826d165 --- /dev/null +++ b/sync-job/overlays/stage-to-prod/kustomization.yaml @@ -0,0 +1,32 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +commonLabels: + app.kubernetes.io/name: thoth + app.kubernetes.io/component: sync-job +resources: + - ../common/ +patches: + - patch: |- + - op: add + path: /spec/workflowSpec/templates/container/0/env/5/value + value: "solver" + target: + version: v1alpha1 + kind: CronWorkflow + name: sync-job-cw + - patch: |- + - op: add + path: /spec/workflowSpec/templates/container/1/env/5/value + value: "resolver" + target: + version: v1alpha1 + kind: CronWorkflow + name: sync-job-cw + - patch: |- + - op: add + path: /spec/workflowSpec/templates/container/2/env/5/value + value: "security-indicator" + target: + version: v1alpha1 + kind: CronWorkflow + name: sync-job-cw