Merge pull request #128 from eth-cscs/dev

Merge to master (version 1.8.1)

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "python.analysis.extraPaths": [
+        "./src/common",
+        "./src/storage"
+    ]
+}

ci/k8s/Jenkinsfile

@@ -0,0 +1,286 @@
+def vault_secrets = [
+                [path: 'firecrest/dev', engineVersion: 2, secretValues: [
+                        [envVar: 'JFROG_API_KEY', vaultKey: 'JFROG_API_KEY'],
+                        [envVar: 'REGISTRY', vaultKey: 'REGISTRY'],
+                        [envVar: 'REGISTRY_GROUP', vaultKey: 'REGISTRY_GROUP'],
+                        [envVar: 'REGISTRY_USER', vaultKey: 'REGISTRY_USER'],
+                        [envVar: 'REPO_PREFIX', vaultKey: 'REPO_PREFIX'],
+                        [envVar: 'K8S_CLUSTER_URL', vaultKey: 'K8S_CLUSTER_URL'],
+                        [envVar: 'firecrestci_github_access_token', vaultKey: 'firecrestci_github_access_token'], 
+                    ]
+                ],
+            ]
+
+def vault_config = [timeout: 10, skipSslVerification: true]
+
+
+
+pipeline {
+    agent {
+        kubernetes {
+            //label 'kaniko'
+            yaml """
+kind: Pod
+metadata:
+  name: kaniko
+spec:
+  containers:
+  - name: kaniko
+    image: gcr.io/kaniko-project/executor:debug
+    imagePullPolicy: Always
+    command:
+    - /busybox/cat
+    tty: true
+"""
+        }
+    }
+    environment {
+        DEPLOY_NAMESPACE = "firecrest-dev"
+        GIT_COMMIT = "${env.GIT_COMMIT}"
+        GIT_COMMIT_SHORT = "${env.GIT_COMMIT.take(7)}"
+    }
+
+    stages {
+        stage("Kaniko Config") {
+
+            steps {
+                container(name: 'kaniko') {
+                    withVault([vaultSecrets: vault_secrets, configuration: vault_config]) {
+
+                        sh '''
+                            mkdir -p /kaniko/.docker
+                            echo '{"auths":{"'$REGISTRY'":{"username":"'$REGISTRY_USER'","password":"'$JFROG_API_KEY'"}}}' > /kaniko/.docker/config.json
+                        '''
+                }                
+            }
+          }
+        }
+
+        stage("Kaniko Build & Push to registry") {
+            steps {
+                container(name: 'kaniko') {
+                    withVault([vaultSecrets: vault_secrets, configuration: vault_config]) {
+
+                        sh '''
+                            #cd ${BUILD_NUMBER}/firecrest
+                            pwd
+                            ls -la
+
+                            #/kaniko/executor --context ./ --dockerfile deploy/docker/base/Dockerfile\
+                            #   --destination $REPO_PREFIX/f7t-base:$GIT_COMMIT_SHORT --cleanup
+
+                            for ms in certificator compute reservations status storage tasks utilities; do
+                                /kaniko/executor --build-arg BASE_IMAGE=$REPO_PREFIX/f7t-base:latest --registry-mirror $REGISTRY \
+                                --context ./ --dockerfile deploy/docker/$ms/Dockerfile --destination $REPO_PREFIX/$ms:$GIT_COMMIT_SHORT --cleanup
+                            done
+
+                            #/kaniko/executor --context deploy/test-build --dockerfile ./cluster/Dockerfile \
+                            #--destination $REPO_PREFIX/cluster:$GIT_COMMIT_SHORT --cleanup
+
+                            /kaniko/executor --context src/tests/template_client --dockerfile ./Dockerfile \
+                            --destination $REPO_PREFIX/client:$GIT_COMMIT_SHORT --cleanup
+
+                            /kaniko/executor --context ./ --dockerfile deploy/docker/tester/Dockerfile \
+                            --destination $REPO_PREFIX/tester:$GIT_COMMIT_SHORT --cleanup
+                        '''
+                    }
+                }
+            }
+        }
+
+        stage("F7T+Infra Pods Deployment") {
+            steps {
+                withVault([vaultSecrets: vault_secrets, configuration: vault_config]) {
+                    withKubeConfig([credentialsId: 'firecrest-cicd-secret', serverUrl: K8S_CLUSTER_URL]) {
+                        sh '''
+                            curl -s -O https://get.helm.sh/helm-v3.7.1-linux-amd64.tar.gz
+                            tar -xvf helm-v3.7.1-linux-amd64.tar.gz
+
+                            export PATH=$PATH:$(pwd)/linux-amd64
+                            helm list -n "$DEPLOY_NAMESPACE"
+                            #cd ${BUILD_NUMBER}/firecrest
+
+                            cd deploy/k8s
+                            ls -la
+                            echo "registry: $REPO_PREFIX\ntag: '$GIT_COMMIT_SHORT'\nnamespace: "$DEPLOY_NAMESPACE"\nregistry_secret_creds: registry-credentials\n" > values-dev.yaml
+
+                            for app in config certificator client compute jaeger keycloak kong minio openapi reservations status storage tasks utilities; do
+                            helm uninstall "$app-env-dev" -n "$DEPLOY_NAMESPACE" || true
+                            helm install  --wait --wait-for-jobs --timeout 60s  "$app-env-dev" $app -n "$DEPLOY_NAMESPACE"  -f values-dev.yaml
+                            done
+                            helm ls -n "$DEPLOY_NAMESPACE"
+                        '''
+                    }
+
+                }
+            }
+        }
+        stage("Cluster Pod Deployment For Microservices Tests") {
+            steps {
+                withVault([vaultSecrets: vault_secrets, configuration: vault_config]) {
+                    withKubeConfig([credentialsId: 'firecrest-cicd-secret', serverUrl: K8S_CLUSTER_URL]) {
+
+                        sh '''
+                            curl -s -O https://get.helm.sh/helm-v3.7.1-linux-amd64.tar.gz
+                            tar -xvf helm-v3.7.1-linux-amd64.tar.gz
+                            export PATH=$PATH:$(pwd)/linux-amd64
+                            helm list -n "$DEPLOY_NAMESPACE"
+                            #cd ${BUILD_NUMBER}/firecrest
+
+                            cd deploy/k8s
+                            ls -la
+
+                            # Cluster is deployed separatelly ALWAYS with tag = tds
+
+                            echo "registry: $REPO_PREFIX\ntag: latest \nnamespace: "$DEPLOY_NAMESPACE"\nregistry_secret_creds: registry-credentials\n" > values-cluster-dev.yaml
+
+                            helm uninstall cluster-env-dev -n "$DEPLOY_NAMESPACE" || true
+                            helm install --wait --timeout 60s  cluster-env-dev cluster -n "$DEPLOY_NAMESPACE"  -f values-cluster-dev.yaml
+
+                            helm ls -n "$DEPLOY_NAMESPACE"
+                        '''
+                    }
+                }
+            }
+        }
+
+        stage("Performing Microservices Tests") {
+            steps {
+                withVault([vaultSecrets: vault_secrets, configuration: vault_config]) {
+                    withKubeConfig([credentialsId: 'firecrest-cicd-secret', serverUrl: K8S_CLUSTER_URL]) {
+                        sh '''
+                            # installing helm
+                            curl -s -O https://get.helm.sh/helm-v3.7.1-linux-amd64.tar.gz
+                            tar -xvf helm-v3.7.1-linux-amd64.tar.gz
+                            export PATH=$PATH:$(pwd)/linux-amd64
+
+                            # installing kubectl
+                            curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+                            chmod +x kubectl
+                            export PATH=$PATH:$(pwd)
+
+                            helm list -n "$DEPLOY_NAMESPACE"
+                            kubectl get pods -n "$DEPLOY_NAMESPACE"
+
+                            #cd ${BUILD_NUMBER}/firecrest
+
+                            cd deploy/k8s
+                            ls -la
+                            echo "registry: $REPO_PREFIX\ntag: '$GIT_COMMIT_SHORT'\nnamespace: "$DEPLOY_NAMESPACE"\nregistry_secret_creds: registry-credentials\n" > values-dev.yaml
+
+                            for use_gateway in True False; do
+
+                                helm uninstall tester-env-dev -n "$DEPLOY_NAMESPACE" || true
+
+                                echo "Test using gateway: $use_gateway"
+                                helm install --wait --timeout 120s tester-env-dev tester -n "$DEPLOY_NAMESPACE"  -f values-dev.yaml --set tag=$GIT_COMMIT_SHORT \
+                                    --set workingDir="/firecrest/src/tests/automated_tests" \
+                                    --set use_gateway="$use_gateway" \
+                                    --set pytest_config_file="firecrest-dev.ini"
+
+                                while :
+                                do
+                                    sleep 20s
+
+                                    tester_pod=$(kubectl get pods --selector=job-name=job-tester -n "$DEPLOY_NAMESPACE" --output=jsonpath='{.items[*].metadata.name}')
+                                    echo "Tester pod is: $tester_pod"
+                                    pdstatus=$(kubectl get pods -n "$DEPLOY_NAMESPACE" $tester_pod -o jsonpath="{.status.phase}")
+
+                                    if [ "$pdstatus" = "Running" ]; then echo "$tester_pod is still $pdstatus"; continue; fi
+                                    kubectl logs $tester_pod -n firecrest-dev
+                                    if [ "$pdstatus" = "Failed" ]; then echo "$tester_pod has $pdstatus"; exit 1; fi
+                                    if [ "$pdstatus" = "Succeeded" ]; then echo "$tester_pod has $pdstatus"; break; fi
+                                done
+
+                            done
+                        '''
+                    }
+                }
+            }
+        }
+        stage('Tag for TDS'){
+            when {
+                branch 'dev-k8s'
+            }
+            steps {
+                withVault([vaultSecrets: vault_secrets, configuration: vault_config]) {
+                    sh  '''
+                        tag="tds"
+                        for ms in certificator client compute reservations status storage tasks utilities; do
+                            img="https://$REGISTRY/artifactory/api/copy/$REGISTRY_GROUP/$ms/$GIT_COMMIT_SHORT?to=/$REGISTRY_GROUP/$ms/$tag"
+                            echo "URL: $img"
+                            response=$(curl -s -o /dev/null -w "%{http_code}" -u $REGISTRY_USER:$JFROG_API_KEY -X POST $img)
+                            if [ "$response" = "200" ]; then echo "Image $ms/$GIT_COMMIT_SHORT copued successfully to $tag  (status_code=$response)"; else echo "Image $ms/$GIT_COMMIT_SHORT couldn't be copied (status_code=$response)"; fi
+                        done
+                    '''
+                }
+
+            }
+        }
+
+    }
+    post{
+        always {
+            withVault([vaultSecrets: vault_secrets, configuration: vault_config]){
+                withKubeConfig([credentialsId: 'firecrest-cicd-secret', serverUrl: K8S_CLUSTER_URL]) {
+                    sh '''
+                        # installing kubectl
+                        curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+                        chmod +x kubectl
+                        export PATH=$PATH:$(pwd)
+
+                        # install helm
+                        curl -s -O https://get.helm.sh/helm-v3.7.1-linux-amd64.tar.gz
+                        tar -xvf helm-v3.7.1-linux-amd64.tar.gz
+                        pwd
+                        export PATH=$PATH:$(pwd)/linux-amd64
+
+                        # getting logs from all pods
+                        pods=$(kubectl get pods -n "$DEPLOY_NAMESPACE" --output=jsonpath='{.items[*].metadata.name}' --selector=app!=tester)
+
+                        for pod in $pods; do
+                            kubectl logs $pod -n "$DEPLOY_NAMESPACE"  --all-containers=true || true
+                        done
+
+                        # removing infrastructure
+                        for app in config certificator client cluster compute jaeger keycloak kong minio openapi reservations status storage tasks tester utilities; do
+                            helm uninstall "$app-env-dev" -n "$DEPLOY_NAMESPACE" || true
+                        done
+
+                        # remove images
+                        for ms in certificator client compute reservations status storage tasks tester utilities; do
+                            img="https://$REGISTRY/artifactory/$REGISTRY_GROUP/$ms/$GIT_COMMIT_SHORT"
+                            response=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 20 -u $REGISTRY_USER:$JFROG_API_KEY -XDELETE $img)
+                            if [ "$response" = "204" ]; then echo "Image $ms/$GIT_COMMIT_SHORT deleted successfully (status_code=$response)"; else echo "Image $ms/$GIT_COMMIT_SHORT couldn't be deleted (status_code=$response)"; fi
+                            sleep 20s
+                        done
+                    '''
+                }            
+            }
+        }
+        success {
+            script {
+                // Notify Github on success
+                withVault([vaultSecrets: vault_secrets, configuration: vault_config]){
+                    sh 'curl -H "Authorization: token ' + "${firecrestci_github_access_token}"  + '" "https://api.github.com/repos/eth-cscs/firecrest/statuses/' + "${env.GIT_COMMIT}" + '" \\' +
+                    '-H "Content-Type: application/json" \\' +
+                    '-X POST \\' +
+                    '-d "{\\"state\\": \\"success\\",\\"context\\": \\"continuous-integration/jenkins\\", \\"description\\": \\"Jenkins\\", \\"target_url\\": \\"' + "${env.BUILD_URL}" + '/console\\"}"'
+                }
+            }
+            slackSend (message: "Feature branch: ${env.BRANCH_NAME} -> ${env.BUILD_DISPLAY_NAME} successful - details: ${env.BUILD_URL}", color: "good")
+        }
+        unsuccessful{
+            script {
+                // Notify Github on failure
+                withVault([vaultSecrets: vault_secrets, configuration: vault_config]){
+                    sh 'curl -H "Authorization: token ' + "${firecrestci_github_access_token}"  + '" "https://api.github.com/repos/eth-cscs/firecrest/statuses/' + "${env.GIT_COMMIT}" + '" \\' +
+                    '-H "Content-Type: application/json" \\' +
+                    '-X POST \\' +
+                    '-d "{\\"state\\": \\"failure\\",\\"context\\": \\"continuous-integration/jenkins\\", \\"description\\": \\"Jenkins\\", \\"target_url\\": \\"' + "${env.BUILD_URL}" + '/console\\"}"'
+                }
+            }
+            slackSend (message: "Feature branch: ${env.BRANCH_NAME} -> ${env.BUILD_DISPLAY_NAME} failed - details: ${env.BUILD_URL}",  color: "danger")
+        }
+    }
+}

ci/pre-prod/Jenkinsfile

@@ -63,23 +63,6 @@ node {
             )
         }
 
-        /*
-        // This should be executed once from awx itself
-        // since the test server is a permanent host (not dynamically created)
-        stage('Provisioning of test environment') {
-
-            // Install docker, python etc
-            ansibleTower(
-                towerServer: 'awx-local',
-                templateType: 'job',
-                jobTemplate: 'Test server provisioning',
-                towerLogLevel: 'full',
-                removeColor: false,
-                verbose: true
-                async: false
-            )
-        }*/
-
         stage('Deploy firecrest in test environment') {
 
             ansibleTower(
@@ -171,6 +154,11 @@ node {
                 towerLogLevel: 'full',
                 removeColor: false,
                 verbose: true,
+                extraVars: """
+                docker_registry_host: 148.187.97.229:5000
+                build_tag: $shortCommit
+                commit_id: $longCommit
+                """,
                 async: false,
                 throwExceptionWhenFail: false
             )

ci/pre-prod/build_image_role/tasks/main.yml

@@ -75,21 +75,10 @@
     state: present
     push: yes  
 
-- name: build client image
-  docker_image:
-    name: "localhost:5000/client:{{ build_tag }}"
-    build:
-      path: /home/firecrest/awx-firecrest-build/src/tests/template_client
-      dockerfile: ./Dockerfile
-      pull: yes
-    source: build
-    state: present
-    push: yes
-
 - name: Pull redis and push to local registry
   docker_image:
     name: redis:5
     repository: "localhost:5000/taskpersistence:{{ build_tag }}"
     push: yes
     source: pull
-
+    

ci/pre-prod/build_image_role/vars/main.yml

@@ -16,6 +16,4 @@ image_definitions:
   tasks:
     build_path: /home/firecrest/awx-firecrest-build
   utilities:
-    build_path: /home/firecrest/awx-firecrest-build
-  openapi:
-    build_path: /home/firecrest/awx-firecrest-build
+    build_path: /home/firecrest/awx-firecrest-build  

ci/pre-prod/deploy_demo_playbook.yml

@@ -158,7 +158,8 @@
         image: "{{ docker_registry_host }}/storage:{{build_tag}}"
         env_file: "{{ firecrest_dir }}/deploy/demo/common/common.env"
         env: 
-          F7T_S3_URL: "http://192.168.220.19:9000"
+          F7T_S3_PRIVATE_URL: "http://192.168.220.19:9000"
+          F7T_S3_PUBLIC_URL: "http://192.168.220.19:9000"
           F7T_S3_ACCESS_KEY: "storage_access_key"
           F7T_S3_SECRET_KEY: "storage_secret_key"
           F7T_STORAGE_POLLING_INTERVAL: "60" 
@@ -217,7 +218,7 @@
     - name: Keycloack
       community.general.docker_container:
         name: fckeycloak
-        image: "jboss/keycloak:4.8.3.Final"
+        image: "jboss/keycloak:9.0.2"
         env_file:  "{{ firecrest_dir }}/deploy/demo/keycloak/keycloak.env" 
         env:
           KEYCLOAK_IMPORT: "/var/tmp/config.json"
@@ -235,7 +236,7 @@
     - name: Kong
       community.general.docker_container:
         name: kong
-        image: "kong:latest"
+        image: "kong:2.5.0"
         env:
           KONG_DATABASE: "off"
           KONG_DECLARATIVE_CONFIG: "/kong.yml"