From 08d6468ccc9d5ef3a843f351335703a3abfdc0f3 Mon Sep 17 00:00:00 2001 From: Chandan Kumar Date: Tue, 18 Jun 2024 12:37:01 +0530 Subject: [PATCH] [WIP] Meta content provider Allowing users to test opendev + github changes together Signed-off-by: Chandan Kumar --- .../meta_content_provider.yml | 142 ++++++++++++++++++ ci/playbooks/meta_content_provider/run.yml | 54 +++++++ ci/templates/projects.yaml | 1 + scenarios/centos-9/meta_content_provider.yml | 23 +++ zuul.d/content_provider.yaml | 21 +++ zuul.d/projects.yaml | 1 + 6 files changed, 242 insertions(+) create mode 100644 ci/playbooks/meta_content_provider/meta_content_provider.yml create mode 100644 ci/playbooks/meta_content_provider/run.yml create mode 100644 scenarios/centos-9/meta_content_provider.yml diff --git a/ci/playbooks/meta_content_provider/meta_content_provider.yml b/ci/playbooks/meta_content_provider/meta_content_provider.yml new file mode 100644 index 0000000000..b91b62e3d1 --- /dev/null +++ b/ci/playbooks/meta_content_provider/meta_content_provider.yml @@ -0,0 +1,142 @@ +--- +- name: Bootstrap step + ansible.builtin.import_playbook: >- + {{ + [ + ansible_user_dir, + zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir, + 'playbooks', + '01-bootstrap.yml' + ] | ansible.builtin.path_join + }} + +- name: "Run ci/playbooks/meta_content_provider/meta_content_provider.yml" + hosts: "{{ cifmw_target_host | default('localhost') }}" + gather_facts: true + tasks: + - name: Install necessary dependencies + ansible.builtin.include_role: + name: 'install_yamls_makes' + tasks_from: 'make_download_tools' + + - name: Build OpenStack Services Packages using DLRN + vars: + cifmw_bop_yum_repos_dir: "{{ cifmw_build_containers_repo_dir }}" + cifmw_bop_gating_repo_dest: "{{ cifmw_build_containers_repo_dir }}" + ansible.builtin.include_role: + name: build_openstack_packages + + - name: Check for gating repo + ansible.builtin.stat: + path: "{{ cifmw_build_containers_repo_dir }}/gating.repo" + register: _gating_repo + + - name: Deploy content provider registry + ansible.builtin.include_role: + name: registry_deploy + + - name: Construct project change list + ansible.builtin.set_fact: + zuul_change_list: "{{ zuul_change_list | default([]) + [item.project.short_name] }}" + cacheable: true + with_items: "{{ zuul['items'] }}" + when: + - zuul is defined + - "'change_url' in item" + + - name: Build openstack services container when gating repo exists + when: + - ("'os-net-config' not in zuul_change_list") and + (_gating_repo.stat.exists) + ansible.builtin.include_role: + name: build_containers + + - name: Build EDPM Images + when: "'edpm-image-builder' in zuul_change_list" + block: + - name: Get latest commit when no PR is provided + ansible.builtin.command: # noqa: command-instead-of-module + cmd: git show-ref --head --hash head + args: + chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/edpm-image-builder" + register: git_head_out + + - name: Set pr_sha to be used as image tag + ansible.builtin.set_fact: + pr_sha: "{{ git_head_out.stdout | trim }}" + cacheable: true + + - name: Build edpm and ipa images + ansible.builtin.include_role: + name: edpm_build_images + vars: + cifmw_edpm_build_images_tag: "{{ pr_sha }}" + + - name: Push edpm-hardened-uefi image to registry + containers.podman.podman_image: + name: "{{ item }}" + push_args: + dest: "{{ cifmw_rp_registry_ip | default('localhost') }}:5001/{{ item }}:{{ pr_sha }}" + tag: "{{ pr_sha }}" + push: true + loop: + - edpm-hardened-uefi + - ironic-python-agent + + - name: Set build images output + ansible.builtin.set_fact: + cifmw_build_images_output: + images: + edpm-hardened-uefi: + image: "{{ cifmw_rp_registry_ip | default('localhost') }}:5001/edpm-hardened-uefi:{{ pr_sha }}" + ironic-python-agent: + image: "{{ cifmw_rp_registry_ip | default('localhost') }}:5001/ironic-python-agent:{{ pr_sha }}" + cacheable: true + + - name: Set build images output + when: "'edpm-image-builder' not in zuul_change_list" + ansible.builtin.set_fact: + cifmw_build_images_output: {} + + - name: Set var for cifmw_operator_build_operators var + # It handles the case of setting image_base for + # openstack-ansibleee-operator and openstack-operator project + # for openstack-ansibleee-operator, it will return openstack-ansibleee + # and for openstack-operator, openstack will be returned + when: + - zuul is defined + - "'project' in zuul" + - "'short_name' in zuul.project" + ansible.builtin.set_fact: + cifmw_operator_build_operators: + - name: "openstack-operator" + src: "~/src/github.com/{{ cifmw_operator_build_org }}/openstack-operator" + image_base: >- + {{ zuul.project.short_name | split('-') | reject('search','operator') | join('-') }} + + - name: Build Operators + ansible.builtin.include_role: + name: operator_build + + - name: Get the containers list from container registry + ansible.builtin.uri: + url: "http://{{ cifmw_rp_registry_ip }}:5001/v2/_catalog" + return_content: true + register: cp_imgs + + - name: Add the container list to file + ansible.builtin.copy: + content: "{{ cp_imgs.content }}" + dest: "{{ ansible_user_dir }}/local_registry.log" + mode: "0644" + +- name: Run log related tasks + ansible.builtin.import_playbook: >- + {{ + [ + ansible_user_dir, + zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir, + 'playbooks', + '99-logs.yml' + ] | ansible.builtin.path_join + }} diff --git a/ci/playbooks/meta_content_provider/run.yml b/ci/playbooks/meta_content_provider/run.yml new file mode 100644 index 0000000000..f3ba25d311 --- /dev/null +++ b/ci/playbooks/meta_content_provider/run.yml @@ -0,0 +1,54 @@ +--- +- name: "Run ci/playbooks/meta_content_provider/run.yml" + hosts: "{{ cifmw_zuul_target_host | default('all') }}" + gather_facts: true + tasks: + - name: Filter out host if needed + when: + - cifmw_zuul_target_host is defined + - cifmw_zuul_target_host != 'all' + - inventory_hostname != cifmw_zuul_target_host + ansible.builtin.meta: end_host + + - name: Deploy Meta content provider + environment: + ANSIBLE_CONFIG: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/ansible.cfg" + ansible.builtin.command: + chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework" + cmd: >- + ansible-playbook ci/playbooks/meta_content_provider/meta_content_provider.yml + -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml" + -e @scenarios/centos-9/base.yml + -e @scenarios/centos-9/meta_content_provider.yml + -e "cifmw_rp_registry_ip={{ cifmw_rp_registry_ip }}" + -e "cifmw_rp_registry_port=5001" + {%- if cifmw_extras is defined %} + {%- for extra_vars in cifmw_extras %} + -e "{{ extra_vars }}" + {%- endfor %} + {%- endif %} + -e "@{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters/zuul-params.yml" + + - name: Include inner ansible vars file + ansible.builtin.slurp: + src: "{{ cifmw_artifacts_basedir }}/artifacts/ansible-vars.yml" + register: _inner_ansible + + - name: Return Zuul Data + ansible.builtin.debug: + msg: >- + Running Content provider registry on + {{ cifmw_rp_registry_ip | default('nowhere') }} + + - name: Set up content registry IP address + vars: + _inner_ansible_vars: "{{ _inner_ansible.content | b64decode | from_yaml }}" + _dlrn_md5: "{{ _inner_ansible_vars.cifmw_repo_setup_full_hash }}" + zuul_return: + data: + zuul: + pause: true + content_provider_registry_ip: "{{ cifmw_rp_registry_ip | default('nowhere') | trim }}" + cifmw_operator_build_output: "{{ _inner_ansible_vars.cifmw_operator_build_output }}" + cifmw_build_images_output: "{{ _inner_ansible_vars.cifmw_build_images_output }}" + content_provider_dlrn_md5_hash: "{{ _dlrn_md5 | default('') }}" diff --git a/ci/templates/projects.yaml b/ci/templates/projects.yaml index f3f85feafb..374c0c8892 100644 --- a/ci/templates/projects.yaml +++ b/ci/templates/projects.yaml @@ -21,4 +21,5 @@ - cifmw-multinode-kuttl - cifmw-tcib - cifmw-architecture-validate-hci + - ci-framework-openstack-meta-content-provider # Start generated content diff --git a/scenarios/centos-9/meta_content_provider.yml b/scenarios/centos-9/meta_content_provider.yml new file mode 100644 index 0000000000..96dd316180 --- /dev/null +++ b/scenarios/centos-9/meta_content_provider.yml @@ -0,0 +1,23 @@ +--- +ansible_user_dir: "{{ lookup('env', 'HOME') }}" +cifmw_installyamls_repos: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/install_yamls" + +# build_operators vars +cifmw_operator_build_push_registry: "{{ cifmw_rp_registry_ip }}:{{ cifmw_rp_registry_port }}" +cifmw_operator_build_push_org: "openstack-k8s-operators" +cifmw_operator_build_org: "openstack-k8s-operators" +cifmw_operator_build_meta_build: true +cifmw_operator_build_local_registry: 1 +cifmw_operator_build_push_registry_tls_verify: false + +# build_containers vars +cifmw_build_containers_tcib_src: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/tcib" +cifmw_repo_setup_src: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/repo-setup" +cifmw_build_containers_repo_dir: "{{ cifmw_basedir }}/artifacts/repositories" +cifmw_build_containers_push_registry: "{{ cifmw_rp_registry_ip }}:{{ cifmw_rp_registry_port }}" +cifmw_build_containers_push_containers: false +cifmw_build_containers_buildah_push: true +cifmw_build_containers_registry_namespace: podified-antelope-centos9 + +# edpm_build_images vars +cifmw_edmp_build_images_push_registry: "{{ cifmw_rp_registry_ip }}:{{ cifmw_rp_registry_port }}" diff --git a/zuul.d/content_provider.yaml b/zuul.d/content_provider.yaml index d4047a39cd..389dd347e0 100644 --- a/zuul.d/content_provider.yaml +++ b/zuul.d/content_provider.yaml @@ -2,3 +2,24 @@ - job: name: content-provider-base parent: openstack-k8s-operators-content-provider + +- job: + name: openstack-meta-content-provider + parent: openstack-k8s-operators-content-provider + description: | + A zuul job to build contents(rpms, openstack services containers, + operators) from opendev and github changes. + timeout: 4000 + run: + - ci/playbooks/meta_content_provider/run.yml + +- job: + name: ci-framework-openstack-meta-content-provider + parent: openstack-meta-content-provider + files: + - ^roles/build_containers/(?!meta|README).* + - ^roles/build_openstack_packages/(?!meta|README).* + - ^roles/registry_deploy/(?!meta|README).* + - ^roles/edpm_build_images/(?!meta|README).* + - ^roles/operator_build/(?!meta|README).* + - ^ci/playbooks/meta_content_provider/.* diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 1b51102754..d0ef1b87cd 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -12,6 +12,7 @@ - cifmw-multinode-kuttl - cifmw-tcib - cifmw-architecture-validate-hci + - ci-framework-openstack-meta-content-provider - cifmw-molecule-artifacts - cifmw-molecule-build_containers - cifmw-molecule-build_openstack_packages