Image sync (#350)

* Minor fix to dockerfile

* Reorganise deployment

Use different dirs for each component with the terraform specifics, try
to have common parts without duplication

* Image syncing

* Improve deployment

* Make sure we have CAs in the right place

* Adjust device

* Add the images to the schema

* Add image sync to WALTON

* Improved role

* Ignore hadolint issue

* New ADR for image sync

* Fine tune deployment

* Update README

* Debug

* Fix deployment call

* Do not debug here

* Remove site configuration that we are not using

* Improve text

* Moved repo to EGI-Federation

* Improve building image

.github/workflows/deploy-cloud-info.yml

@@ -0,0 +1,18 @@
+---
+name: 'Deploy cloud-info'
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - "deploy/**"
+
+jobs:
+  deploy:
+    uses: ./.github/workflows/deploy.yml
+    with:
+      dir: "deploy/cloud-info"
+      tags: "cloud-info,docker"
+    secrets: inherit

.github/workflows/deploy-image-sync.yml

@@ -0,0 +1,19 @@
+---
+name: 'Deploy image sync'
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - "deploy/**"
+
+jobs:
+  deploy:
+    uses: ./.github/workflows/deploy.yml
+    with:
+      dir: "deploy/image-sync"
+      tags: "docker,image-sync"
+    secrets: inherit
+

.github/workflows/deploy.yml

@@ -1,13 +1,15 @@
 ---
-name: 'Deploy'
+name: Deploy
 
 on:
-  push:
-    branches:
-      - main
-  pull_request:
-    paths:
-      - "deploy/**"
+  workflow_call:
+    inputs:
+      dir:
+        required: true
+        type: string
+      tags:
+        required: true
+        type: string
 
 jobs:
   terraform:
@@ -38,7 +40,8 @@ jobs:
                             -d "grant_type=refresh_token&client_id=token-portal&scope=$SCOPE&refresh_token=$REFRESH_TOKEN" \
                           | jq -r ".access_token")
           echo "::add-mask::$OIDC_TOKEN"
-          cd deploy
+          cd "${{ inputs.dir }}"
+          cp ../clouds.yaml .
           BACKEND_SITE="$(yq -r .clouds.backend.site clouds.yaml)"
           BACKEND_VO="$(yq -r .clouds.backend.vo clouds.yaml)"
           EGI_SITE="$(yq -r .clouds.deploy.site clouds.yaml)"
@@ -67,19 +70,21 @@ jobs:
       - name: Terraform Format
         id: fmt
         run: |
-          cd deploy
+          cd "${{ inputs.dir }}"
           terraform fmt -check
       - name: Terraform init
         id: init
         run: |
-          cd deploy
+          cd "${{ inputs.dir }}"
           terraform init
       - name: Adjust cloud-init file
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
         run: |
-          cd deploy
+          cd "${{ inputs.dir }}"
+          cat ../cloud-init.yaml extra-cloud-init.yaml > cloud-init.yaml
           sed -i -e "s/%TOKEN%/${{ secrets.GITHUB_TOKEN }}/" cloud-init.yaml
+          sed -i -e "s/%TAGS%/${{ inputs.tags }}/" cloud-init.yaml
           sed -i -e "s/%REF%/${{ github.sha }}/" cloud-init.yaml
           sed -i -e "s/%SHORT_REF%/$(git rev-parse --short HEAD)/" cloud-init.yaml
           sed -i -e "s#%SLACK_WEBHOOK_URL%#$SLACK_WEBHOOK_URL#" cloud-init.yaml
@@ -89,7 +94,7 @@ jobs:
         id: plan
         if: github.event_name == 'pull_request'
         run: |
-          cd deploy
+          cd "${{ inputs.dir }}"
           terraform plan -no-color -var-file="$EGI_SITE.tfvars"
         continue-on-error: true
       - name: Update Pull Request
@@ -125,13 +130,13 @@ jobs:
         id: terraform-apply
         if: github.ref == 'refs/heads/main' && github.event_name == 'push'
         run: |
-          cd deploy
+          cd "${{ inputs.dir }}"
           terraform apply -auto-approve -var-file="$EGI_SITE.tfvars"
       - name: Get VM ID
         id: terraform-vm-id
         if: github.ref == 'refs/heads/main' && github.event_name == 'push'
         run: |
-          cd deploy
+          cd "${{ inputs.dir }}"
           terraform output -raw instance-id
       - name: Re-configure providers access
         env:
@@ -145,8 +150,8 @@ jobs:
                             -d "grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=token-portal&scope=$SCOPE" \
                           | jq -r ".access_token")
           echo "::add-mask::$OIDC_TOKEN"
-          cd deploy
-          git checkout -- clouds.yaml
+          cd "${{ inputs.dir }}"
+          cp ../clouds.yaml .
           BACKEND_SITE="$(yq -r .clouds.backend.site clouds.yaml)"
           BACKEND_VO="$(yq -r .clouds.backend.vo clouds.yaml)"
           BACKEND_OS_TOKEN="$(fedcloud openstack token issue --oidc-access-token "$OIDC_TOKEN" \
@@ -164,13 +169,13 @@ jobs:
           max_attempts: 20
           retry_wait_seconds: 40
           command: >
-            pushd deploy &&
+            pushd "${{ inputs.dir }}" &&
             openstack --os-cloud backend --os-token "$BACKEND_OS_TOKEN" object save fedcloud-catchall "${{ steps.terraform-vm-id.outputs.stdout }}" &&
             openstack --os-cloud backend --os-token "$BACKEND_OS_TOKEN" object delete fedcloud-catchall "${{ steps.terraform-vm-id.outputs.stdout }}"
       - name: Look for errors
         if: github.ref == 'refs/heads/main' && github.event_name == 'push'
         run: |
-          cd deploy
+          cd "${{ inputs.dir }}"
           # show the status in the build log
           cat "${{ steps.terraform-vm-id.outputs.stdout }}"
           grep -v "error" "${{ steps.terraform-vm-id.outputs.stdout }}"

README.md

@@ -11,6 +11,16 @@ its configuration with a format as follows:
 ```yaml
 gocdb: <name in gocdb of the site>
 endpoint: <keystone endpoint of the site>
+# optional: use central image sync
+images:
+  # true, get sync, false do not
+  sync: true
+  # a list of supported formats of the site can be specified
+  # if not available, no conversion will be done, so whatever format
+  # is available in AppDB will be used
+  formats:
+    - qcow2
+    - raw
 # optionally specify a protocol for the Keystone V3 federation API
 protocol: openid | oidc (default is openid)
 # optionally specify a region name if using different regions
@@ -26,30 +36,17 @@ vos:
     publicNetwork: <name of the public network>
 ```
 
-## Generating configurations
-
-The mapping configuration of the VOs supported at each site can be easily
-generated with the `generate-config.py` utility (requires `pyyaml`). It takes as
-parameter the YAML file describing the site and will dump the requested
-keystone, caso or cloudkeeper-os json config:
-
-```shell
-python generate-config.py --config-type keystone sites/SITE.yaml
-```
-
-This mapping file should work for most cases. If you have special requirements
-open an issue so we can tune the generation to meet your needs!
-
 ## Docker containers
 
 Components are run as docker containers, which if not available upstream, are
 generated in this repository.
 
 ## Deployment
 
-Deployment is managed on a separate private repository that includes several
-secrets. Deployment is done with ansible using a
-[dedicated role](./deploy/roles/catchall) with:
+Deployment is managed with GitHub actions, there is a VM for the
+cloud-info-provider and one VM for the image sync. Check the [deploy](./deploy)
+directory for details. Configuration is done with ansible using a
+[dedicated role](./deploy/roles/catchall):
 
 ```sh
 ansible-playbook -i inventory.yaml --extra-vars "@secrets.yaml" playbook.yaml
@@ -60,5 +57,5 @@ where:
 - `inventory.yaml` contains the ansible inventory with the host to configure
 - `secrets.yaml` contains the credentials for every configured VO and a valid
   token for the AMS
-- `playbook.yaml` is an ansible playbook that just uses the
-  `fedcloud-catchall-ops` role to configure the host
+- `playbook.yaml` is an ansible playbook that just uses the `catchall` role to
+  configure the host

cloud-info/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3 as build
+FROM python:3 AS build
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 

deploy/cloud-init.yaml

@@ -36,14 +36,15 @@ write_files:
       COMMIT_SHA="%REF%"
       SHORT_COMMIT_SHA="%SHORT_REF%"
       FEDCLOUD_LOCKER_TOKEN="%FEDCLOUD_LOCKER_TOKEN%"
+      TAGS="%TAGS%"
 
       # get the repo code and untar at cwd
       curl -L -H "Accept: application/vnd.github.v3+raw" \
               "https://api.github.com/repos/EGI-Federation/fedcloud-catchall-operations/tarball/$COMMIT_SHA" | \
               tar xz --strip=1
       cd deploy
       ./deploy.sh "$OAUTH_TOKEN" "$COMMIT_SHA" "$FEDCLOUD_LOCKER_TOKEN" \
-                  "$SHORT_COMMIT_SHA" "$SLACK_WEBHOOK_URL"
+                  "$TAGS" "$SHORT_COMMIT_SHA" "$SLACK_WEBHOOK_URL"
     path: /var/lib/cloud/scripts/per-boot/deploy.sh
     permissions: '0755'
   - content: |

deploy/deploy.sh

@@ -4,29 +4,37 @@
 # - a GitHub OAUTH_TOKEN to update the PR
 # - the COMMIT_SHA
 # - a locker for fedcloud secret to obtain the secrets
+# - tags for the ansible configuration
 # - the SHORT_SHA used for pulling the docker image to use
 # - a SLACK_WEBHOOK_URL to report on the status
 set -e
 
 OAUTH_TOKEN="$1"
 COMMIT_SHA="$2"
 FEDCLOUD_SECRET_LOCKER="$3"
-SHORT_SHA="$4"
-SLACK_WEBHOOK_URL="$5"
+TAGS="$4"
+SHORT_SHA="$5"
+SLACK_WEBHOOK_URL="$6"
 
 # create a virtual env for fedcloudclient
 python3 -m venv "$PWD/.venv"
 "$PWD/.venv/bin/pip" install fedcloudclient
 
+TMP_SECRETS="$(mktemp)"
 "$PWD/.venv/bin/fedcloud" secret get --locker-token "$FEDCLOUD_SECRET_LOCKER" \
-	deploy data >secrets.yaml
+	deploy data >"$TMP_SECRETS" && mv "$TMP_SECRETS" secrets.yaml
 
-echo "cloud_info_image: \"ghcr.io/egi-federation/fedcloud-cloud-info:sha-$SHORT_SHA\"" >>extra-vars.yaml
+cat >>extra-vars.yaml <<EOF
+cloud_info_image: "ghcr.io/egi-federation/fedcloud-cloud-info:sha-$SHORT_SHA"
+image_sync_image: "ghcr.io/egi-federation/fedcloud-image-sync:sha-$SHORT_SHA"
+site_config_dir: "$(readlink -f ../sites)"
+EOF
 
 # Configure!
 if ansible-playbook -i inventory.yaml \
 	--extra-vars @secrets.yaml \
 	--extra-vars @extra-vars.yaml \
+	--tags "$TAGS" \
 	playbook.yaml >ansible.log 2>&1; then
 	status_summary="success"
 	color="#6DBF59"
@@ -91,7 +99,7 @@ cat >slack_body.json <<EOF
           "type": "section",
           "text": {
             "type": "mrkdwn",
-            "text": "fedcloud-catchall-operations deployment was completed for <$comment_url| PR \`#$ISSUE_NUMBER\`> "
+            "text": "fedcloud-catchall deployment was completed for <$comment_url| PR \`#$ISSUE_NUMBER\`> "
           }
         }
       ]

deploy/image-sync/NCG-INGRID-PT.tfvars

@@ -0,0 +1,8 @@
+# Network
+net_id = "f15a0e1f-570e-4135-9739-a59b8c2b3e8e"
+
+# Flavor: svc1.m 2cores/4GB RAM
+flavor_id = "737f8483-8063-4567-a8e5-e09a4bcbdb49"
+
+# Image: ubuntu 22.04
+image_id = "966f2e5a-7b48-4cb2-be92-6e2132413cf2"

deploy/image-sync/backend.tf

@@ -0,0 +1,12 @@
+# This is where the info about the deployment is to be stored
+terraform {
+  backend "swift" {
+    container = "terraform-image-sync"
+    cloud     = "backend"
+  }
+}
+
+# The provider where the deployment is actually performed
+provider "openstack" {
+  cloud = "deploy"
+}

deploy/image-sync/extra-cloud-init.yaml

@@ -0,0 +1,13 @@
+# Disk layout
+disk_setup:
+  /dev/sdb:
+    table_type: 'mbr'
+    layout: true
+    overwrite: false
+fs_setup:
+  - filesystem: ext4
+    device: /dev/sdb
+    partition: any
+    overwrite: false
+mounts:
+  - [ /dev/sdb, /var/cache/image-sync ]

deploy/image-sync/main.tf

@@ -0,0 +1,26 @@
+resource "openstack_blockstorage_volume_v3" "image-cache" {
+  name = "image-cache"
+  size = 200
+}
+
+resource "openstack_compute_instance_v2" "image-sync" {
+  name            = "image-sync"
+  image_id        = var.image_id
+  flavor_id       = var.flavor_id
+  security_groups = ["default"]
+  user_data       = file("cloud-init.yaml")
+  network {
+    uuid = var.net_id
+  }
+}
+
+resource "openstack_compute_volume_attach_v2" "attached" {
+  instance_id = openstack_compute_instance_v2.image-sync.id
+  volume_id   = openstack_blockstorage_volume_v3.image-cache.id
+}
+
+
+
+output "instance-id" {
+  value = openstack_compute_instance_v2.image-sync.id
+}

deploy/image-sync/vars.tf

@@ -0,0 +1,14 @@
+variable "net_id" {
+  type        = string
+  description = "The id of the network"
+}
+
+variable "image_id" {
+  type        = string
+  description = "VM image id"
+}
+
+variable "flavor_id" {
+  type        = string
+  description = "VM flavor id"
+}

deploy/image-sync/versions.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_providers {
+    openstack = {
+      source  = "terraform-provider-openstack/openstack"
+      version = "~> 1.48"
+    }
+  }
+  required_version = ">= 0.13"
+}

deploy/playbook.yaml

@@ -3,7 +3,5 @@
   become: true
   roles:
     - role: catchall
-      tags: ["all", "docker"]
       vars:
         site_config_dir: ../sites/
-        checkin_token_endpoint: https://aai.egi.eu/auth/realms/egi/protocol/openid-connect/token