Fixes 3517: job to retry existing failed tasks (#556)

Fixes 3517: job to retry failed tasks

cmd/retry_failed_tasks/main.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"os"
+
+	"github.com/content-services/content-sources-backend/pkg/config"
+	"github.com/content-services/content-sources-backend/pkg/db"
+	"github.com/rs/zerolog/log"
+)
+
+func main() {
+	args := os.Args
+	config.Load()
+	config.ConfigureLogging()
+	err := db.Connect()
+
+	if err != nil {
+		log.Panic().Err(err).Msg("Failed to connect to database")
+	}
+
+	if len(args) < 2 || args[1] != "--force" {
+		log.Fatal().Msg("Requires arguments: --force")
+	}
+
+	query :=
+		`
+			UPDATE tasks
+			SET next_retry_time = statement_timestamp(), retries = 0
+			WHERE started_at IS NOT NULL AND finished_at IS NOT NULL 
+		  		AND status = 'failed' AND type = 'delete-repository-snapshots';
+		`
+	result := db.DB.Exec(query)
+	if result.Error != nil {
+		log.Fatal().Err(result.Error).Msg("Could not update failed tasks.")
+	} else {
+		log.Warn().Msgf("Updated %v tasks", result.RowsAffected)
+	}
+}

deployments/deployment.yaml

@@ -266,6 +266,60 @@ objects:
               - mountPath: /tmp
                 name: tmpdir
       jobs:
+        - name: retry-failed-tasks
+          podSpec:
+            securityContext:
+              runAsNonRoot: true
+              runAsUser: 1001
+            image: ${IMAGE}:${IMAGE_TAG}
+            inheritEnv: true
+            command:
+              - /retry_failed_tasks
+              - --force
+            env:
+              - name: CLOWDER_ENABLED
+                value: ${CLOWDER_ENABLED}
+              - name: RH_CDN_CERT_PAIR
+                valueFrom:
+                  secretKeyRef:
+                    name: content-sources-certs
+                    key: cdn.redhat.com
+              - name: SENTRY_DSN
+                valueFrom:
+                  secretKeyRef:
+                    name: content-sources-sentry
+                    key: dsn
+                    optional: true
+              - name: CLIENTS_PULP_SERVER
+                value: ${{CLIENTS_PULP_SERVER}}
+              - name: CLIENTS_PULP_DOWNLOAD_POLICY
+                value: ${{CLIENTS_PULP_DOWNLOAD_POLICY}}
+              - name: CLIENTS_PULP_USERNAME
+                value: ${{CLIENTS_PULP_USERNAME}}
+              - name: CLIENTS_PULP_PASSWORD
+                valueFrom:
+                  secretKeyRef:
+                    name: pulp-content-sources-password
+                    key: password
+                    optional: true
+              - name: LOGGING_LEVEL
+                value: ${{LOGGING_LEVEL}}
+              - name: NEW_TASKING_SYSTEM
+                value: ${NEW_TASKING_SYSTEM}
+              - name: FEATURES_SNAPSHOTS_ENABLED
+                value: ${FEATURES_SNAPSHOTS_ENABLED}
+              - name: FEATURES_SNAPSHOTS_ACCOUNTS
+                value: ${FEATURES_SNAPSHOTS_ACCOUNTS}
+              - name: FEATURES_ADMIN_TASKS_ENABLED
+                value: ${FEATURES_ADMIN_TASKS_ENABLED}
+              - name: FEATURES_ADMIN_TASKS_ACCOUNTS
+                value: ${FEATURES_ADMIN_TASKS_ACCOUNTS}
+              - name: CLIENTS_RBAC_BASE_URL
+                value: ${{CLIENTS_RBAC_BASE_URL}}
+              - name: OPTIONS_ALWAYS_RUN_CRON_TASKS
+                value: ${OPTIONS_ALWAYS_RUN_CRON_TASKS}
+              - name: OPTIONS_ENABLE_NOTIFICATIONS
+                value: ${OPTIONS_ENABLE_NOTIFICATIONS}
         - name: repair-redhat
           podSpec:
             securityContext:

deployments/jobs.yaml

@@ -23,3 +23,13 @@ objects:
     appName: content-sources-backend
     jobs:
       - repair-redhat
+- apiVersion: cloud.redhat.com/v1alpha1
+  kind: ClowdJobInvocation
+  metadata:
+    labels:
+      app: content-sources-backend
+    name: retry-failed-tasks-2024-02-01
+  spec:
+    appName: content-sources-backend
+    jobs:
+      - retry-failed-tasks