[WIP] Meta content provider

Allowing users to test opendev + github changes together

Signed-off-by: Chandan Kumar <[email protected]>

ci/playbooks/meta_content_provider/meta_content_provider.yml

@@ -0,0 +1,142 @@
+---
+- name: Bootstrap step
+  ansible.builtin.import_playbook: >-
+    {{
+        [
+          ansible_user_dir,
+          zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir,
+          'playbooks',
+          '01-bootstrap.yml'
+        ] | ansible.builtin.path_join
+    }}
+
+- name: "Run ci/playbooks/meta_content_provider/meta_content_provider.yml"
+  hosts: "{{ cifmw_target_host | default('localhost') }}"
+  gather_facts: true
+  tasks:
+    - name: Install necessary dependencies
+      ansible.builtin.include_role:
+        name: 'install_yamls_makes'
+        tasks_from: 'make_download_tools'
+
+    - name: Build OpenStack Services Packages using DLRN
+      vars:
+        cifmw_bop_yum_repos_dir: "{{ cifmw_build_containers_repo_dir }}"
+        cifmw_bop_gating_repo_dest: "{{ cifmw_build_containers_repo_dir }}"
+      ansible.builtin.include_role:
+        name: build_openstack_packages
+
+    - name: Check for gating repo
+      ansible.builtin.stat:
+        path: "{{ cifmw_build_containers_repo_dir }}/gating.repo"
+      register: _gating_repo
+
+    - name: Deploy content provider registry
+      ansible.builtin.include_role:
+        name: registry_deploy
+
+    - name: Construct project change list
+      ansible.builtin.set_fact:
+        zuul_change_list: "{{ zuul_change_list | default([]) + [item.project.short_name] }}"
+        cacheable: true
+      with_items: "{{ zuul['items'] }}"
+      when:
+        - zuul is defined
+        - "'change_url' in item"
+
+    - name: Build openstack services container when gating repo exists
+      when:
+        - ("'os-net-config' not in zuul_change_list") and
+          (_gating_repo.stat.exists)
+      ansible.builtin.include_role:
+        name: build_containers
+
+    - name: Build EDPM Images
+      when: "'edpm-image-builder' in zuul_change_list"
+      block:
+        - name: Get latest commit when no PR is provided
+          ansible.builtin.command:  # noqa: command-instead-of-module
+            cmd: git show-ref --head --hash head
+          args:
+            chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/edpm-image-builder"
+          register: git_head_out
+
+        - name: Set pr_sha to be used as image tag
+          ansible.builtin.set_fact:
+            pr_sha: "{{ git_head_out.stdout | trim }}"
+            cacheable: true
+
+        - name: Build edpm and ipa images
+          ansible.builtin.include_role:
+            name: edpm_build_images
+          vars:
+            cifmw_edpm_build_images_tag: "{{ pr_sha }}"
+
+        - name: Push edpm-hardened-uefi image to registry
+          containers.podman.podman_image:
+            name: "{{ item }}"
+            push_args:
+              dest: "{{ cifmw_rp_registry_ip | default('localhost') }}:5001/{{ item }}:{{ pr_sha }}"
+            tag: "{{ pr_sha }}"
+            push: true
+          loop:
+            - edpm-hardened-uefi
+            - ironic-python-agent
+
+        - name: Set build images output
+          ansible.builtin.set_fact:
+            cifmw_build_images_output:
+              images:
+                edpm-hardened-uefi:
+                  image: "{{ cifmw_rp_registry_ip | default('localhost') }}:5001/edpm-hardened-uefi:{{ pr_sha }}"
+                ironic-python-agent:
+                  image: "{{ cifmw_rp_registry_ip | default('localhost') }}:5001/ironic-python-agent:{{ pr_sha }}"
+            cacheable: true
+
+    - name: Set build images output when EDPM image is not built
+      when: "'edpm-image-builder' not in zuul_change_list"
+      ansible.builtin.set_fact:
+        cifmw_build_images_output: {}
+
+    - name: Set var for cifmw_operator_build_operators var
+      # It handles the case of setting image_base for
+      # openstack-ansibleee-operator and openstack-operator project
+      # for openstack-ansibleee-operator, it will return openstack-ansibleee
+      # and for openstack-operator, openstack will be returned
+      when:
+        - zuul is defined
+        - "'project' in zuul"
+        - "'short_name' in zuul.project"
+      ansible.builtin.set_fact:
+        cifmw_operator_build_operators:
+          - name: "openstack-operator"
+            src: "~/src/github.com/{{ cifmw_operator_build_org }}/openstack-operator"
+            image_base: >-
+              {{ zuul.project.short_name | split('-') | reject('search','operator') | join('-') }}
+
+    - name: Build Operators
+      ansible.builtin.include_role:
+        name: operator_build
+
+    - name: Get the containers list from container registry
+      ansible.builtin.uri:
+        url: "http://{{ cifmw_rp_registry_ip }}:5001/v2/_catalog"
+        return_content: true
+      register: cp_imgs
+
+    - name: Add the container list to file
+      ansible.builtin.copy:
+        content: "{{ cp_imgs.content }}"
+        dest: "{{ ansible_user_dir }}/local_registry.log"
+        mode: "0644"
+
+- name: Run log related tasks
+  ansible.builtin.import_playbook: >-
+    {{
+        [
+          ansible_user_dir,
+          zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir,
+          'playbooks',
+          '99-logs.yml'
+        ] | ansible.builtin.path_join
+    }}

ci/playbooks/meta_content_provider/run.yml

@@ -0,0 +1,54 @@
+---
+- name: "Run ci/playbooks/meta_content_provider/run.yml"
+  hosts: "{{ cifmw_zuul_target_host | default('all') }}"
+  gather_facts: true
+  tasks:
+    - name: Filter out host if needed
+      when:
+        - cifmw_zuul_target_host is defined
+        - cifmw_zuul_target_host != 'all'
+        - inventory_hostname != cifmw_zuul_target_host
+      ansible.builtin.meta: end_host
+
+    - name: Deploy Meta content provider
+      environment:
+        ANSIBLE_CONFIG: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/ansible.cfg"
+      ansible.builtin.command:
+        chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
+        cmd: >-
+          ansible-playbook ci/playbooks/meta_content_provider/meta_content_provider.yml
+          -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"
+          -e @scenarios/centos-9/base.yml
+          -e @scenarios/centos-9/meta_content_provider.yml
+          -e "cifmw_rp_registry_ip={{ cifmw_rp_registry_ip }}"
+          -e "cifmw_rp_registry_port=5001"
+          {%- if cifmw_extras is defined %}
+          {%-   for extra_vars in cifmw_extras %}
+          -e "{{   extra_vars }}"
+          {%-   endfor %}
+          {%- endif %}
+          -e "@{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters/zuul-params.yml"
+
+    - name: Include inner ansible vars file
+      ansible.builtin.slurp:
+        src: "{{ cifmw_artifacts_basedir }}/artifacts/ansible-vars.yml"
+      register: _inner_ansible
+
+    - name: Return Zuul Data
+      ansible.builtin.debug:
+        msg: >-
+          Running Content provider registry on
+          {{ cifmw_rp_registry_ip | default('nowhere') }}
+
+    - name: Set up content registry IP address
+      vars:
+        _inner_ansible_vars: "{{ _inner_ansible.content | b64decode | from_yaml }}"
+        _dlrn_md5: "{{ _inner_ansible_vars.cifmw_repo_setup_full_hash }}"
+      zuul_return:
+        data:
+          zuul:
+            pause: true
+          content_provider_registry_ip: "{{ cifmw_rp_registry_ip | default('nowhere') | trim }}"
+          cifmw_operator_build_output: "{{ _inner_ansible_vars.cifmw_operator_build_output }}"
+          cifmw_build_images_output: "{{ _inner_ansible_vars.cifmw_build_images_output }}"
+          content_provider_dlrn_md5_hash: "{{ _dlrn_md5 | default('') }}"

ci/templates/projects.yaml

@@ -21,4 +21,5 @@
         - cifmw-multinode-kuttl
         - cifmw-tcib
         - cifmw-architecture-validate-hci
+        - ci-framework-openstack-meta-content-provider
 # Start generated content

roles/edpm_prepare/tasks/main.yml

@@ -134,7 +134,9 @@
   ansible.builtin.set_fact:
     cifmw_update_containers_edpm_image_url: "{{ cifmw_build_images_output['images']['edpm-hardened-uefi']['image'] }}"
     cacheable: true
-  when: cifmw_build_images_output is defined
+  when: 
+    - cifmw_build_images_output is defined
+    - cifmw_build_images_output.keys() | length > 0
 
 # Prepare and kustomize the OpenStackControlPlane CR
 - name: Prepare OpenStack control plane CR

scenarios/centos-9/meta_content_provider.yml

@@ -0,0 +1,24 @@
+---
+ansible_user_dir: "{{ lookup('env', 'HOME') }}"
+cifmw_installyamls_repos: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/install_yamls"
+
+# build_operators vars
+cifmw_operator_build_push_registry: "{{ cifmw_rp_registry_ip }}:{{ cifmw_rp_registry_port }}"
+cifmw_operator_build_push_org: "openstack-k8s-operators"
+cifmw_operator_build_org: "openstack-k8s-operators"
+cifmw_operator_build_meta_build: true
+cifmw_operator_build_local_registry: 1
+cifmw_operator_build_push_registry_tls_verify: false
+
+# build_containers vars
+cifmw_build_containers_tcib_src: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/tcib"
+cifmw_repo_setup_src: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/repo-setup"
+cifmw_build_containers_repo_dir: "{{ cifmw_basedir }}/artifacts/repositories"
+cifmw_build_containers_push_registry: "{{ cifmw_rp_registry_ip }}:{{ cifmw_rp_registry_port }}"
+cifmw_build_containers_push_containers: false
+cifmw_build_containers_buildah_push: true
+cifmw_build_containers_registry_namespace: podified-antelope-centos9
+cifmw_build_containers_image_tag: "{{ cifmw_repo_setup_full_hash }}"
+
+# edpm_build_images vars
+cifmw_edmp_build_images_push_registry: "{{ cifmw_rp_registry_ip }}:{{ cifmw_rp_registry_port }}"

zuul.d/content_provider.yaml

@@ -2,3 +2,24 @@
 - job:
     name: content-provider-base
     parent: openstack-k8s-operators-content-provider
+
+- job:
+    name: openstack-meta-content-provider
+    parent: openstack-k8s-operators-content-provider
+    description: |
+      A zuul job to build contents(rpms, openstack services containers,
+      operators) from opendev and github changes.
+    timeout: 4000
+    run:
+      - ci/playbooks/meta_content_provider/run.yml
+
+- job:
+    name: ci-framework-openstack-meta-content-provider
+    parent: openstack-meta-content-provider
+    files:
+      - ^roles/build_containers/(?!meta|README).*
+      - ^roles/build_openstack_packages/(?!meta|README).*
+      - ^roles/registry_deploy/(?!meta|README).*
+      - ^roles/edpm_build_images/(?!meta|README).*
+      - ^roles/operator_build/(?!meta|README).*
+      - ^ci/playbooks/meta_content_provider/.*

zuul.d/projects.yaml

@@ -12,6 +12,7 @@
       - cifmw-multinode-kuttl
       - cifmw-tcib
       - cifmw-architecture-validate-hci
+      - ci-framework-openstack-meta-content-provider
       - cifmw-molecule-artifacts
       - cifmw-molecule-build_containers
       - cifmw-molecule-build_openstack_packages