Add Meta content provider job to test opendev and github changes

Meta content provider does following things:
- It combines 3 content provider (operator, tcib and edpm image build)
  into one content provider to test all changes together.
- If the changes coming from opendev, tcib and os-net-config, then it
  will build the DLRN packages and create gating repo.
- If there is a gating and non os-net-config changes, then it will do
  the tcib build.
- If there is a edpm-image-build change, then it will perform the image
  build.
- All the built content will be pushed on single registry with different
  namespace.
- The content provider will expose the same zuul return vars to
  dependent job to pick proper built content.
- It also returns content_provider_os_registry_url var to dependent job
  and improve edpm_prepare role to set proper vars.
- Generate gating repo only when there are repos.

Note: It make changes to edpm_prepare to check edpm_image_build_output
key length. If no edpm image is built by meta content provider then
edpm_image_build_output will return an empty dict.

Signed-off-by: Chandan Kumar <[email protected]>

ci/playbooks/meta_content_provider/meta_content_provider.yml

@@ -0,0 +1,149 @@
+---
+- name: Bootstrap step
+  ansible.builtin.import_playbook: >-
+    {{
+        [
+          ansible_user_dir,
+          zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir,
+          'playbooks',
+          '01-bootstrap.yml'
+        ] | ansible.builtin.path_join
+    }}
+
+- name: Run ci/playbooks/meta_content_provider/meta_content_provider.yml
+  hosts: "{{ cifmw_target_host | default('localhost') }}"
+  gather_facts: true
+  tasks:
+    - name: Install necessary dependencies
+      ansible.builtin.include_role:
+        name: 'install_yamls_makes'
+        tasks_from: 'make_download_tools'
+
+    - name: Build OpenStack Services Packages using DLRN
+      vars:
+        cifmw_bop_yum_repos_dir: "{{ cifmw_build_containers_repo_dir }}"
+        cifmw_bop_gating_repo_dest: "{{ cifmw_build_containers_repo_dir }}"
+      ansible.builtin.include_role:
+        name: build_openstack_packages
+
+    - name: Check for gating repo
+      ansible.builtin.stat:
+        path: "{{ cifmw_build_containers_repo_dir }}/gating.repo"
+      register: _gating_repo
+
+    - name: Deploy content provider registry
+      ansible.builtin.include_role:
+        name: registry_deploy
+
+    - name: Construct project change list
+      ansible.builtin.set_fact:
+        zuul_change_list: "{{ zuul_change_list | default([]) + [item.project.short_name] }}"
+        cacheable: true
+      with_items: "{{ zuul['items'] }}"
+      when:
+        - zuul is defined
+        - "'change_url' in item"
+
+    - name: Build openstack services container when gating repo exists
+      when:
+        - "'os-net-config' not in zuul_change_list"
+        - _gating_repo.stat.exists
+      block:
+        - name: Build OpenStack services containers
+          ansible.builtin.include_role:
+            name: build_containers
+
+        - name: Return registry_url
+          ansible.builtin.set_fact:
+            content_provider_os_registry_url: "{{ cifmw_build_containers_push_registry }}/{{ cifmw_build_containers_registry_namespace }}"
+
+    - name: Build EDPM Images
+      when: "'edpm-image-builder' in zuul_change_list"
+      block:
+        - name: Get latest commit when no PR is provided
+          ansible.builtin.command:  # noqa: command-instead-of-module
+            cmd: git show-ref --head --hash head
+          args:
+            chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/edpm-image-builder"
+          register: git_head_out
+
+        - name: Set pr_sha to be used as image tag
+          ansible.builtin.set_fact:
+            pr_sha: "{{ git_head_out.stdout | trim }}"
+            cacheable: true
+
+        - name: Build edpm and ipa images
+          ansible.builtin.include_role:
+            name: edpm_build_images
+          vars:
+            cifmw_edpm_build_images_tag: "{{ pr_sha }}"
+
+        - name: Push edpm-hardened-uefi image to registry
+          containers.podman.podman_image:
+            name: "{{ item }}"
+            push_args:
+              dest: "{{ cifmw_rp_registry_ip | default('localhost') }}:5001/{{ item }}:{{ pr_sha }}"
+            tag: "{{ pr_sha }}"
+            push: true
+          loop:
+            - edpm-hardened-uefi
+            - ironic-python-agent
+
+        - name: Set build images output
+          ansible.builtin.set_fact:
+            cifmw_build_images_output:
+              images:
+                edpm-hardened-uefi:
+                  image: "{{ cifmw_rp_registry_ip | default('localhost') }}:5001/edpm-hardened-uefi:{{ pr_sha }}"
+                ironic-python-agent:
+                  image: "{{ cifmw_rp_registry_ip | default('localhost') }}:5001/ironic-python-agent:{{ pr_sha }}"
+            cacheable: true
+
+    - name: Set build images output when EDPM image is not built
+      when: "'edpm-image-builder' not in zuul_change_list"
+      ansible.builtin.set_fact:
+        cifmw_build_images_output: {}
+
+    - name: Set var for cifmw_operator_build_operators var
+      # It handles the case of setting image_base for
+      # openstack-ansibleee-operator and openstack-operator project
+      # for openstack-ansibleee-operator, it will return openstack-ansibleee
+      # and for openstack-operator, openstack will be returned
+      when:
+        - zuul is defined
+        - "'project' in zuul"
+        - "'short_name' in zuul.project"
+        - "'operator' in zuul.project.short_name"
+      ansible.builtin.set_fact:
+        cifmw_operator_build_operators:
+          - name: "openstack-operator"
+            src: "~/src/github.com/{{ cifmw_operator_build_org }}/openstack-operator"
+            image_base: >-
+              {{ zuul.project.short_name | split('-') | reject('search','operator') | join('-') }}
+
+    - name: Build Operators
+      ansible.builtin.include_role:
+        name: operator_build
+
+    - name: Get the containers list from container registry
+      ansible.builtin.uri:
+        url: "http://{{ cifmw_rp_registry_ip }}:5001/v2/_catalog"
+        return_content: true
+      register: cp_imgs
+
+    - name: Add the container list to file
+      ansible.builtin.copy:
+        content: "{{ cp_imgs.content }}"
+        dest: "{{ ansible_user_dir }}/local_registry.log"
+        mode: "0644"
+
+- name: Run log related tasks
+  ansible.builtin.import_playbook: >-
+    {{
+        [
+          ansible_user_dir,
+          zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir,
+          'playbooks',
+          '99-logs.yml'
+        ] | ansible.builtin.path_join
+    }}

ci/playbooks/meta_content_provider/run.yml

@@ -0,0 +1,61 @@
+---
+- name: "Run ci/playbooks/meta_content_provider/run.yml"
+  hosts: "{{ cifmw_zuul_target_host | default('all') }}"
+  gather_facts: true
+  tasks:
+    - name: Filter out host if needed
+      when:
+        - cifmw_zuul_target_host is defined
+        - cifmw_zuul_target_host != 'all'
+        - inventory_hostname != cifmw_zuul_target_host
+      ansible.builtin.meta: end_host
+
+    - name: Deploy Meta content provider
+      environment:
+        ANSIBLE_CONFIG: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/ansible.cfg"
+      ansible.builtin.command:
+        chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
+        cmd: >-
+          ansible-playbook ci/playbooks/meta_content_provider/meta_content_provider.yml
+          -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"
+          -e @scenarios/centos-9/base.yml
+          -e @scenarios/centos-9/meta_content_provider.yml
+          -e "cifmw_rp_registry_ip={{ cifmw_rp_registry_ip }}"
+          -e "cifmw_rp_registry_port=5001"
+          {%- if cifmw_extras is defined %}
+          {%-   for extra_vars in cifmw_extras %}
+          -e "{{   extra_vars }}"
+          {%-   endfor %}
+          {%- endif %}
+          -e "@{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters/zuul-params.yml"
+
+    - name: Include inner ansible vars file
+      ansible.builtin.slurp:
+        src: "{{ cifmw_artifacts_basedir }}/artifacts/ansible-vars.yml"
+      register: _inner_ansible
+
+    - name: Return Zuul Data
+      ansible.builtin.debug:
+        msg: >-
+          Running Content provider registry on
+          {{ cifmw_rp_registry_ip | default('nowhere') }}
+
+    - name: Return data for dependent job
+      vars:
+        _inner_ansible_vars: "{{ _inner_ansible.content | b64decode | from_yaml }}"
+        _dlrn_md5: "{{ _inner_ansible_vars.cifmw_repo_setup_full_hash }}"
+        _tcib_registry: >-
+          {%- if _inner_ansible_vars.content_provider_os_registry_url is defined -%}
+          {{ _inner_ansible_vars.content_provider_os_registry_url }}
+          {%- else -%}
+          null
+          {%- endif -%}
+      zuul_return:
+        data:
+          zuul:
+            pause: true
+          content_provider_registry_ip: "{{ cifmw_rp_registry_ip | default('nowhere') | trim }}"
+          cifmw_operator_build_output: "{{ _inner_ansible_vars.cifmw_operator_build_output }}"
+          cifmw_build_images_output: "{{ _inner_ansible_vars.cifmw_build_images_output }}"
+          content_provider_dlrn_md5_hash: "{{ _dlrn_md5 | default('') }}"
+          content_provider_os_registry_url: "{{ _tcib_registry | trim }}"

ci/templates/projects.yaml

@@ -21,4 +21,5 @@
         - cifmw-multinode-kuttl
         - cifmw-tcib
         - cifmw-architecture-validate-hci
+        - ci-framework-openstack-meta-content-provider
 # Start generated content

roles/build_openstack_packages/tasks/main.yml

@@ -20,8 +20,5 @@
 - name: Parse Zuul changes and Build RPM packages
   ansible.builtin.import_tasks: parse_and_build_pkgs.yml
 
-- name: Create repo
-  ansible.builtin.import_tasks: create_repo.yml
-
 - name: Cleanup dlrn
   ansible.builtin.import_tasks: cleanup_dlrn.yml

roles/build_openstack_packages/tasks/parse_and_build_pkgs.yml

@@ -50,3 +50,7 @@
       branch: "{{ cifmw_bop_openstack_release }}"
       project: "{{ cifmw_bop_openstack_project_path | basename }}"
   ansible.builtin.include_tasks: run_dlrn.yml
+
+- name: Create repo
+  when: cifmw_bop_change_list | length > 0
+  ansible.builtin.import_tasks: create_repo.yml

roles/edpm_prepare/tasks/kustomize_and_deploy.yml

@@ -74,6 +74,16 @@
         ).metadata.name
       }}
 
+- name: Set vars related to update_containers content provider
+  when:
+    - content_provider_os_registry_url is defined
+    - content_provider_os_registry_url != 'null'
+  ansible.builtin.set_fact:
+    cifmw_update_containers_registry: "{{ content_provider_os_registry_url  | split('/') | first }}"
+    cifmw_update_containers_org: "{{ content_provider_os_registry_url  | split('/') | last }}"
+    cifmw_update_containers_tag: "{{ content_provider_dlrn_md5_hash }}"
+    cifmw_update_containers_openstack: true
+
 - name: Update Openstack containers or BM CSV or Ansibleee CSV to update proper image
   when: >-
       (cifmw_update_containers_edpm_image_url is defined) or

roles/edpm_prepare/tasks/main.yml

@@ -134,7 +134,9 @@
   ansible.builtin.set_fact:
     cifmw_update_containers_edpm_image_url: "{{ cifmw_build_images_output['images']['edpm-hardened-uefi']['image'] }}"
     cacheable: true
-  when: cifmw_build_images_output is defined
+  when:
+    - cifmw_build_images_output is defined
+    - cifmw_build_images_output.keys() | length > 0
 
 # Prepare and kustomize the OpenStackControlPlane CR
 - name: Prepare OpenStack control plane CR

scenarios/centos-9/meta_content_provider.yml

@@ -0,0 +1,24 @@
+---
+ansible_user_dir: "{{ lookup('env', 'HOME') }}"
+cifmw_installyamls_repos: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/install_yamls"
+
+# build_operators vars
+cifmw_operator_build_push_registry: "{{ cifmw_rp_registry_ip }}:{{ cifmw_rp_registry_port }}"
+cifmw_operator_build_push_org: "openstack-k8s-operators"
+cifmw_operator_build_org: "openstack-k8s-operators"
+cifmw_operator_build_meta_build: true
+cifmw_operator_build_local_registry: 1
+cifmw_operator_build_push_registry_tls_verify: false
+
+# build_containers vars
+cifmw_build_containers_tcib_src: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/tcib"
+cifmw_repo_setup_src: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/repo-setup"
+cifmw_build_containers_repo_dir: "{{ cifmw_basedir }}/artifacts/repositories"
+cifmw_build_containers_push_registry: "{{ cifmw_rp_registry_ip }}:{{ cifmw_rp_registry_port }}"
+cifmw_build_containers_push_containers: false
+cifmw_build_containers_buildah_push: true
+cifmw_build_containers_registry_namespace: podified-antelope-centos9
+cifmw_build_containers_image_tag: "{{ cifmw_repo_setup_full_hash }}"
+
+# edpm_build_images vars
+cifmw_edmp_build_images_push_registry: "{{ cifmw_rp_registry_ip }}:{{ cifmw_rp_registry_port }}"

zuul.d/content_provider.yaml

@@ -2,3 +2,24 @@
 - job:
     name: content-provider-base
     parent: openstack-k8s-operators-content-provider
+
+- job:
+    name: openstack-meta-content-provider
+    parent: openstack-k8s-operators-content-provider
+    description: |
+      A zuul job to build content (rpms, openstack services containers,
+      operators) from opendev and github changes.
+    timeout: 5000
+    run:
+      - ci/playbooks/meta_content_provider/run.yml
+
+- job:
+    name: ci-framework-openstack-meta-content-provider
+    parent: openstack-meta-content-provider
+    files:
+      - ^roles/build_containers/(?!meta|README).*
+      - ^roles/build_openstack_packages/(?!meta|README).*
+      - ^roles/registry_deploy/(?!meta|README).*
+      - ^roles/edpm_build_images/(?!meta|README).*
+      - ^roles/operator_build/(?!meta|README).*
+      - ^ci/playbooks/meta_content_provider/.*

zuul.d/projects.yaml

@@ -12,6 +12,7 @@
       - cifmw-multinode-kuttl
       - cifmw-tcib
       - cifmw-architecture-validate-hci
+      - ci-framework-openstack-meta-content-provider
       - cifmw-molecule-artifacts
       - cifmw-molecule-build_containers
       - cifmw-molecule-build_openstack_packages