First version of the image sync script

.adr-dir

@@ -0,0 +1 @@
+doc/architecture/decisions

.github/CODE_OF_CONDUCT.md

@@ -68,17 +68,17 @@ offensive, or harmful.
 
 This Code of Conduct applies both within project spaces and in public spaces
 when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
+representing a project or community include using an official project email
 address, posting via an official social media account, or acting as an appointed
 representative at an online or offline event. Representation of a project may be
 further defined and clarified by project maintainers.
 
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the EGI Foundation team at [email protected]. The team will
-review and investigate all complaints, and will respond in a way that it deems
-appropriate to the circumstances. The team is obligated to maintain
+reported by contacting the [EGI Foundation team](mailto:[email protected]). The
+team will review and investigate all complaints, and will respond in a way that
+it deems appropriate to the circumstances. The team is obligated to maintain
 confidentiality with regard to the reporter of an incident. Further details of
 specific enforcement policies may be posted separately.
 

.github/workflows/deploy.yml → .github/workflows/cloud-info-deploy.yml

@@ -7,7 +7,7 @@ on:
       - main
   pull_request:
     paths:
-      - "deploy/**"
+      - "/cloud-info/deploy/**"
 
 jobs:
   terraform:
@@ -25,20 +25,20 @@ jobs:
           curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 > jq
           chmod +x jq
           pip install yq git+https://github.com/tdviet/fedcloudclient.git
-          curl -L https://github.com/oidc-mytoken/client/releases/download/v0.3.0/mytoken_0.3.0_Linux_x86_64.tar.gz \
-            | tar -xzf -
-          mkdir ~/.mytoken
-          curl https://raw.githubusercontent.com/oidc-mytoken/client/master/config/example-config.yaml > ~/.mytoken/config.yaml
       - name: Configure providers access
         env:
-          MYTOKEN: ${{ secrets.MYTOKEN }}
           REFRESH_TOKEN: ${{ secrets.REFRESH_TOKEN }}
+          ANSIBLE_SECRETS: ${{ secrets.ANSIBLE_SECRETS }}
         run: |
+          # using parametric scopes to only have access to cloud.egi.eu VO
+          SCOPE="openid%20email%20profile%20voperson_id"
+          SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:cloud.egi.eu:role=vm_operator#aai.egi.eu"
+          SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:cloud.egi.eu:role=member#aai.egi.eu"
           OIDC_TOKEN=$(curl -X POST "https://aai.egi.eu/auth/realms/egi/protocol/openid-connect/token" \
-                            -d "grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=token-portal&scope=openid%20email%20profile%20voperson_id%20eduperson_entitlement" \
+                            -d "grant_type=refresh_token&client_id=token-portal&scope=$SCOPE&refresh_token=$REFRESH_TOKEN" \
                           | jq -r ".access_token")
           echo "::add-mask::$OIDC_TOKEN"
-          cd deploy
+          cd cloud-info/deploy
           BACKEND_SITE="$(yq -r .clouds.backend.site clouds.yaml)"
           BACKEND_VO="$(yq -r .clouds.backend.vo clouds.yaml)"
           EGI_SITE="$(yq -r .clouds.deploy.site clouds.yaml)"
@@ -54,39 +54,42 @@ jobs:
           sed -i -e "s/deploy_secret/$DEPLOY_OS_TOKEN/" clouds.yaml
           mkdir -p ~/.config/openstack
           touch ~/.config/openstack/secure.yaml
+          FEDCLOUD_LOCKER_TOKEN="$(fedcloud secret locker create \
+                                   --oidc-access-token "$OIDC_TOKEN" \
+                                   --ttl 1h --num-uses 2)"
+          echo "::add-mask::$FEDCLOUD_LOCKER_TOKEN"
+          fedcloud secret put --locker-token "$FEDCLOUD_LOCKER_TOKEN" deploy "data=$ANSIBLE_SECRETS"
+          echo "FEDCLOUD_LOCKER_TOKEN=$FEDCLOUD_LOCKER_TOKEN" >> "$GITHUB_ENV"
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v3
         with:
           terraform_version: 1.2.9
       - name: Terraform Format
         id: fmt
         run: |
-          cd deploy
+          cd cloud-info/deploy
           terraform fmt -check
       - name: Terraform init
         id: init
         run: |
-          cd deploy
+          cd cloud-info/deploy
           terraform init
       - name: Adjust cloud-init file
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-          ANSIBLE_SECRETS: ${{ secrets.ANSIBLE_SECRETS }}
         run: |
-          cd deploy
+          cd cloud-info/deploy
           sed -i -e "s/%TOKEN%/${{ secrets.GITHUB_TOKEN }}/" cloud-init.yaml
           sed -i -e "s/%REF%/${{ github.sha }}/" cloud-init.yaml
           sed -i -e "s/%SHORT_REF%/$(git rev-parse --short HEAD)/" cloud-init.yaml
           sed -i -e "s#%SLACK_WEBHOOK_URL%#$SLACK_WEBHOOK_URL#" cloud-init.yaml
-          ANSIBLE_ENCODED_SECRETS="$(echo "$ANSIBLE_SECRETS" | base64 -w 0)"
-          echo "::add-mask::$ANSIBLE_ENCODED_SECRETS"
-          sed -i -e "s/%ANSIBLE_SECRETS%/$ANSIBLE_ENCODED_SECRETS/" cloud-init.yaml
+          sed -i -e "s/%FEDCLOUD_LOCKER_TOKEN%/$FEDCLOUD_LOCKER_TOKEN/" cloud-init.yaml
           sed -i -e "s/%CLOUDS_YAML%/$(base64 -w 0 < clouds.yaml)/" cloud-init.yaml
       - name: terraform plan
         id: plan
         if: github.event_name == 'pull_request'
         run: |
-          cd deploy
+          cd cloud-info/deploy
           terraform plan -no-color -var-file="$EGI_SITE.tfvars"
         continue-on-error: true
       - name: Update Pull Request
@@ -122,29 +125,34 @@ jobs:
         id: terraform-apply
         if: github.ref == 'refs/heads/main' && github.event_name == 'push'
         run: |
-          cd deploy
+          cd cloud-info/deploy
           terraform apply -auto-approve -var-file="$EGI_SITE.tfvars"
       - name: Get VM ID
         id: terraform-vm-id
         if: github.ref == 'refs/heads/main' && github.event_name == 'push'
         run: |
-          cd deploy
+          cd cloud-info/deploy
           terraform output -raw instance-id
       - name: Re-configure providers access
         env:
-          MYTOKEN: ${{ secrets.MYTOKEN }}
           REFRESH_TOKEN: ${{ secrets.REFRESH_TOKEN }}
         run: |
+          # using parametric scopes to only have access to cloud.egi.eu VO
+          SCOPE="openid%20email%20profile%20voperson_id"
+          SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:cloud.egi.eu:role=vm_operator#aai.egi.eu"
+          SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:cloud.egi.eu:role=member#aai.egi.eu"
           OIDC_TOKEN=$(curl -X POST "https://aai.egi.eu/auth/realms/egi/protocol/openid-connect/token" \
-                            -d "grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=token-portal&scope=openid%20email%20profile%20voperson_id%20eduperson_entitlement" \
+                            -d "grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=token-portal&scope=$SCOPE" \
                           | jq -r ".access_token")
           echo "::add-mask::$OIDC_TOKEN"
-          cd deploy
+          cd cloud-info/deploy
+          git checkout -- clouds.yaml
           BACKEND_SITE="$(yq -r .clouds.backend.site clouds.yaml)"
           BACKEND_VO="$(yq -r .clouds.backend.vo clouds.yaml)"
           BACKEND_OS_TOKEN="$(fedcloud openstack token issue --oidc-access-token "$OIDC_TOKEN" \
                                                              --site "$BACKEND_SITE" --vo "$BACKEND_VO" -j | jq -r '.[0].Result.id')"
           echo "::add-mask::$BACKEND_OS_TOKEN"
+          echo "BACKEND_OS_TOKEN=$BACKEND_OS_TOKEN" >> "$GITHUB_ENV"
           sed -i -e "s/backend_secret/$BACKEND_OS_TOKEN/" clouds.yaml
           mkdir -p ~/.config/openstack
           touch ~/.config/openstack/secure.yaml
@@ -156,14 +164,13 @@ jobs:
           max_attempts: 20
           retry_wait_seconds: 40
           command: >
-            set -x &&
-            pushd deploy &&
-            openstack --os-cloud backend object save fedcloud-catchall "${{ steps.terraform-vm-id.outputs.stdout }}" &&
-            openstack --os-cloud backend object delete fedcloud-catchall "${{ steps.terraform-vm-id.outputs.stdout }}"
+            pushd cloud-info/deploy &&
+            openstack --os-cloud backend --os-token "$BACKEND_OS_TOKEN" object save fedcloud-catchall "${{ steps.terraform-vm-id.outputs.stdout }}" &&
+            openstack --os-cloud backend --os-token "$BACKEND_OS_TOKEN" object delete fedcloud-catchall "${{ steps.terraform-vm-id.outputs.stdout }}"
       - name: Look for errors
         if: github.ref == 'refs/heads/main' && github.event_name == 'push'
         run: |
-          cd deploy
+          cd cloud-info/deploy
           # show the status in the build log
           cat "${{ steps.terraform-vm-id.outputs.stdout }}"
           grep -v "error" "${{ steps.terraform-vm-id.outputs.stdout }}"

.github/workflows/docker.yml

@@ -17,6 +17,7 @@ jobs:
         image:
           - cloud-info
           - caso
+          - atrope
 
     steps:
       - name: Checkout
@@ -40,7 +41,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
         if: github.event_name != 'pull_request'
       - name: Build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.docker_meta.outputs.tags }}

.github/workflows/image-sync-deploy.yml

@@ -0,0 +1,176 @@
+---
+name: 'Deploy'
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - "/image-sync/deploy/**"
+
+jobs:
+  terraform:
+    name: 'Terraform'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - name: Install environment
+        run: |
+          curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 > jq
+          chmod +x jq
+          pip install yq git+https://github.com/tdviet/fedcloudclient.git
+      - name: Configure providers access
+        env:
+          REFRESH_TOKEN: ${{ secrets.REFRESH_TOKEN }}
+          ANSIBLE_SECRETS: ${{ secrets.ANSIBLE_SECRETS }}
+        run: |
+          # using parametric scopes to only have access to cloud.egi.eu VO
+          SCOPE="openid%20email%20profile%20voperson_id"
+          SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:cloud.egi.eu:role=vm_operator#aai.egi.eu"
+          SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:cloud.egi.eu:role=member#aai.egi.eu"
+          OIDC_TOKEN=$(curl -X POST "https://aai.egi.eu/auth/realms/egi/protocol/openid-connect/token" \
+                            -d "grant_type=refresh_token&client_id=token-portal&scope=$SCOPE&refresh_token=$REFRESH_TOKEN" \
+                          | jq -r ".access_token")
+          echo "::add-mask::$OIDC_TOKEN"
+          cd image-sync/deploy
+          BACKEND_SITE="$(yq -r .clouds.backend.site clouds.yaml)"
+          BACKEND_VO="$(yq -r .clouds.backend.vo clouds.yaml)"
+          EGI_SITE="$(yq -r .clouds.deploy.site clouds.yaml)"
+          DEPLOY_VO="$(yq -r .clouds.deploy.vo clouds.yaml)"
+          echo "EGI_SITE=$EGI_SITE" >> "$GITHUB_ENV"
+          BACKEND_OS_TOKEN="$(fedcloud openstack token issue --oidc-access-token "$OIDC_TOKEN" \
+                                                             --site "$BACKEND_SITE" --vo "$BACKEND_VO" -j | jq -r '.[0].Result.id')"
+          echo "::add-mask::$BACKEND_OS_TOKEN"
+          sed -i -e "s/backend_secret/$BACKEND_OS_TOKEN/" clouds.yaml
+          DEPLOY_OS_TOKEN="$(fedcloud openstack token issue --oidc-access-token "$OIDC_TOKEN" \
+                                                             --site "$EGI_SITE" --vo "$DEPLOY_VO" -j | jq -r '.[0].Result.id')"
+          echo "::add-mask::$DEPLOY_OS_TOKEN"
+          sed -i -e "s/deploy_secret/$DEPLOY_OS_TOKEN/" clouds.yaml
+          mkdir -p ~/.config/openstack
+          touch ~/.config/openstack/secure.yaml
+          FEDCLOUD_LOCKER_TOKEN="$(fedcloud secret locker create \
+                                   --oidc-access-token "$OIDC_TOKEN" \
+                                   --ttl 1h --num-uses 2)"
+          echo "::add-mask::$FEDCLOUD_LOCKER_TOKEN"
+          fedcloud secret put --locker-token "$FEDCLOUD_LOCKER_TOKEN" deploy "data=$ANSIBLE_SECRETS"
+          echo "FEDCLOUD_LOCKER_TOKEN=$FEDCLOUD_LOCKER_TOKEN" >> "$GITHUB_ENV"
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.2.9
+      - name: Terraform Format
+        id: fmt
+        run: |
+          cd image-sync/deploy
+          terraform fmt -check
+      - name: Terraform init
+        id: init
+        run: |
+          cd image-sync/deploy
+          terraform init
+      - name: Adjust cloud-init file
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+        run: |
+          cd image-sync/deploy
+          sed -i -e "s/%TOKEN%/${{ secrets.GITHUB_TOKEN }}/" cloud-init.yaml
+          sed -i -e "s/%REF%/${{ github.sha }}/" cloud-init.yaml
+          sed -i -e "s/%SHORT_REF%/$(git rev-parse --short HEAD)/" cloud-init.yaml
+          sed -i -e "s#%SLACK_WEBHOOK_URL%#$SLACK_WEBHOOK_URL#" cloud-init.yaml
+          sed -i -e "s/%FEDCLOUD_LOCKER_TOKEN%/$FEDCLOUD_LOCKER_TOKEN/" cloud-init.yaml
+          sed -i -e "s/%CLOUDS_YAML%/$(base64 -w 0 < clouds.yaml)/" cloud-init.yaml
+      - name: terraform plan
+        id: plan
+        if: github.event_name == 'pull_request'
+        run: |
+          cd image-sync/deploy
+          terraform plan -no-color -var-file="$EGI_SITE.tfvars"
+        continue-on-error: true
+      - name: Update Pull Request
+        uses: actions/github-script@v7
+        if: github.event_name == 'pull_request'
+        env:
+          PLAN: "terraform\n${{ steps.plan.outputs.stdout }}"
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const output = `#### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\`
+            #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\`
+            #### Terraform Plan 📖\`${{ steps.plan.outcome }}\`
+            <details><summary>Show Plan</summary>
+
+            \`\`\`
+            ${process.env.PLAN}
+            \`\`\`
+
+            </details>
+
+            *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`;
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: output
+            })
+      - name: Terraform Plan Status
+        if: steps.plan.outcome == 'failure'
+        run: exit 1
+      - name: Terraform Apply
+        id: terraform-apply
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        run: |
+          cd image-sync/deploy
+          terraform apply -auto-approve -var-file="$EGI_SITE.tfvars"
+      - name: Get VM ID
+        id: terraform-vm-id
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        run: |
+          cd image-sync/deploy
+          terraform output -raw instance-id
+      - name: Re-configure providers access
+        env:
+          REFRESH_TOKEN: ${{ secrets.REFRESH_TOKEN }}
+        run: |
+          # using parametric scopes to only have access to cloud.egi.eu VO
+          SCOPE="openid%20email%20profile%20voperson_id"
+          SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:cloud.egi.eu:role=vm_operator#aai.egi.eu"
+          SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:cloud.egi.eu:role=member#aai.egi.eu"
+          OIDC_TOKEN=$(curl -X POST "https://aai.egi.eu/auth/realms/egi/protocol/openid-connect/token" \
+                            -d "grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=token-portal&scope=$SCOPE" \
+                          | jq -r ".access_token")
+          echo "::add-mask::$OIDC_TOKEN"
+          cd image-sync/deploy
+          git checkout -- clouds.yaml
+          BACKEND_SITE="$(yq -r .clouds.backend.site clouds.yaml)"
+          BACKEND_VO="$(yq -r .clouds.backend.vo clouds.yaml)"
+          BACKEND_OS_TOKEN="$(fedcloud openstack token issue --oidc-access-token "$OIDC_TOKEN" \
+                                                             --site "$BACKEND_SITE" --vo "$BACKEND_VO" -j | jq -r '.[0].Result.id')"
+          echo "::add-mask::$BACKEND_OS_TOKEN"
+          echo "BACKEND_OS_TOKEN=$BACKEND_OS_TOKEN" >> "$GITHUB_ENV"
+          sed -i -e "s/backend_secret/$BACKEND_OS_TOKEN/" clouds.yaml
+          mkdir -p ~/.config/openstack
+          touch ~/.config/openstack/secure.yaml
+      - name: Get the status file from swift
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 10
+          max_attempts: 20
+          retry_wait_seconds: 40
+          command: >
+            pushd image-sync/deploy &&
+            openstack --os-cloud backend --os-token "$BACKEND_OS_TOKEN" object save fedcloud-catchall "${{ steps.terraform-vm-id.outputs.stdout }}" &&
+            openstack --os-cloud backend --os-token "$BACKEND_OS_TOKEN" object delete fedcloud-catchall "${{ steps.terraform-vm-id.outputs.stdout }}"
+      - name: Look for errors
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        run: |
+          cd image-sync/deploy
+          # show the status in the build log
+          cat "${{ steps.terraform-vm-id.outputs.stdout }}"
+          grep -v "error" "${{ steps.terraform-vm-id.outputs.stdout }}"

.github/workflows/lint.yml

@@ -1,7 +1,7 @@
 ---
 name: Lint
 
-on: [pull_request, push]
+on: pull_request
 
 jobs:
   super-lint:
@@ -12,9 +12,15 @@ jobs:
       # Checks out a copy of your repository on the ubuntu-latest machine
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          # Full git history needed to get proper list of changed files
+          fetch-depth: 0
 
       # Runs the Super-Linter action
       - name: Run Super-Linter
-        uses: github/super-linter/slim@v5
+        uses: github/super-linter/slim@v6
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # Disabling these two for the moment, should be enabled later
+          VALIDATE_CHECKOV: false
+          VALIDATE_PYTHON_PYLINT: false