From 5a42edd0418fa358fed4b11bceee8cf9b4813bb3 Mon Sep 17 00:00:00 2001 From: Sylvain Bauza Date: Wed, 9 Oct 2024 20:51:49 +0200 Subject: [PATCH] [Draft] Split nvidia-mdev arch in two stages --- automation/vars/nvidia-mdev.yaml | 31 ++++-- .../control-plane/kustomization.yaml | 98 +++++++++++++++++++ .../{ => control-plane}/nncp/.gitignore | 0 .../nncp/kustomization.yaml | 2 +- .../{ => control-plane}/nncp/values.yaml | 0 .../control-plane/service-values.yaml | 44 +++++++++ .../nvidia-mdev/edpm-post-driver/.gitignore | 2 + .../edpm-post-driver/deployment/.gitignore | 1 + .../deployment/kustomization.yaml | 47 +++++++++ .../edpm-post-driver/deployment/values.yaml | 16 +++ .../edpm/nodeset/kustomization.yaml | 62 ++++++++++++ .../va/nvidia-mdev/edpm/nodeset/values.yaml | 8 +- examples/va/nvidia-mdev/kustomization.yaml | 13 --- examples/va/nvidia-mdev/service-values.yaml | 9 -- .../deployment/kustomization.yaml | 21 ++++ .../edpm/nodeset/kustomization.yaml | 35 +++---- 16 files changed, 335 insertions(+), 54 deletions(-) create mode 100644 examples/va/nvidia-mdev/control-plane/kustomization.yaml rename examples/va/nvidia-mdev/{ => control-plane}/nncp/.gitignore (100%) rename examples/va/nvidia-mdev/{ => control-plane}/nncp/kustomization.yaml (94%) rename examples/va/nvidia-mdev/{ => control-plane}/nncp/values.yaml (100%) create mode 100644 examples/va/nvidia-mdev/control-plane/service-values.yaml create mode 100644 examples/va/nvidia-mdev/edpm-post-driver/.gitignore create mode 100644 examples/va/nvidia-mdev/edpm-post-driver/deployment/.gitignore create mode 100644 examples/va/nvidia-mdev/edpm-post-driver/deployment/kustomization.yaml create mode 100644 examples/va/nvidia-mdev/edpm-post-driver/deployment/values.yaml delete mode 100644 examples/va/nvidia-mdev/kustomization.yaml delete mode 100644 examples/va/nvidia-mdev/service-values.yaml create mode 100644 va/nvidia-mdev/edpm-post-driver/deployment/kustomization.yaml diff --git a/automation/vars/nvidia-mdev.yaml b/automation/vars/nvidia-mdev.yaml index ef58b3a3c..976f6bb44 100644 --- a/automation/vars/nvidia-mdev.yaml +++ b/automation/vars/nvidia-mdev.yaml @@ -2,7 +2,7 @@ vas: nvidia-mdev: stages: - - path: examples/va/nvidia-mdev/nncp + - path: examples/va/nvidia-mdev/control-plane/nncp wait_conditions: - >- oc -n openstack wait nncp @@ -14,7 +14,7 @@ vas: src_file: values.yaml build_output: nncp.yaml - - path: examples/va/nvidia-mdev + - path: examples/va/nvidia-mdev/control-plane wait_conditions: - >- oc -n openstack wait osctlplane controlplane --for condition=Ready @@ -36,12 +36,6 @@ vas: - name: edpm-nodeset-values src_file: values.yaml build_output: nodeset.yaml - post_stage_run: - - name: Install nvidia driver - type: playbook - # As a reminder, the job needs to set the nvidia driver URL - source: "../../playbooks/nvidia-mdev.yml" - inventory: "${HOME}/ci-framework-data/artifacts/zuul_inventory.yml" - path: examples/va/nvidia-mdev/edpm/deployment wait_conditions: @@ -53,3 +47,24 @@ vas: - name: edpm-deployment-values src_file: values.yaml build_output: deployment.yaml + #post_stage_run: + #- name: Run phase 1 playbook + # type: playbook + # # As a reminder, the job needs to set the nvidia driver URL + # source: "../../playbooks/nvidia-mdev-phase1.yml" + # inventory: "${HOME}/ci-framework-data/artifacts/zuul_inventory.yml" + #- name: Run phase 2 playbook + # type: playbook + # source: "../../playbooks/nvidia-mdev-phase2.yml" + # inventory: "${HOME}/ci-framework-data/artifacts/zuul_inventory.yml" + + - path: examples/va/nvidia-mdev/edpm-post-driver/deployment + wait_conditions: + - >- + oc -n openstack wait + osdpd edpm-deployment-post-driver --for condition=Ready + --timeout=60m + values: + - name: edpm-post-driver-deployment-values + src_file: values.yaml + build_output: deployment-post-driver.yaml diff --git a/examples/va/nvidia-mdev/control-plane/kustomization.yaml b/examples/va/nvidia-mdev/control-plane/kustomization.yaml new file mode 100644 index 000000000..df12f5069 --- /dev/null +++ b/examples/va/nvidia-mdev/control-plane/kustomization.yaml @@ -0,0 +1,98 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +transformers: + # Set namespace to OpenStack on all namespaced objects without a namespace + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../../../lib/networking/metallb + - ../../../../lib/networking/netconfig + - ../../../../lib/networking/nad + - ../../../../lib/control-plane + +resources: + - nncp/values.yaml + - service-values.yaml + +replacements: + # Control plane customization + - source: + kind: ConfigMap + name: service-values + fieldPath: data.neutron.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.neutron.template.customServiceConfig + options: + create: true + # OVN control plane SRIOV customization + - source: + kind: ConfigMap + name: service-values + fieldPath: data.ovn.ovnController.nicMappings + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.ovn.template.ovnController.nicMappings + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.glance.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.glance.template.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.glance.default.replicas + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.glance.template.glanceAPIs.default.replicas + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.swift.enabled + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.swift.enabled + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.schedulerServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.schedulerServiceTemplate.customServiceConfig + options: + create: true diff --git a/examples/va/nvidia-mdev/nncp/.gitignore b/examples/va/nvidia-mdev/control-plane/nncp/.gitignore similarity index 100% rename from examples/va/nvidia-mdev/nncp/.gitignore rename to examples/va/nvidia-mdev/control-plane/nncp/.gitignore diff --git a/examples/va/nvidia-mdev/nncp/kustomization.yaml b/examples/va/nvidia-mdev/control-plane/nncp/kustomization.yaml similarity index 94% rename from examples/va/nvidia-mdev/nncp/kustomization.yaml rename to examples/va/nvidia-mdev/control-plane/nncp/kustomization.yaml index 213659db6..c92bc55fa 100644 --- a/examples/va/nvidia-mdev/nncp/kustomization.yaml +++ b/examples/va/nvidia-mdev/control-plane/nncp/kustomization.yaml @@ -18,7 +18,7 @@ transformers: create: true components: - - ../../../../lib/nncp + - ../../../../../lib/nncp resources: - values.yaml diff --git a/examples/va/nvidia-mdev/nncp/values.yaml b/examples/va/nvidia-mdev/control-plane/nncp/values.yaml similarity index 100% rename from examples/va/nvidia-mdev/nncp/values.yaml rename to examples/va/nvidia-mdev/control-plane/nncp/values.yaml diff --git a/examples/va/nvidia-mdev/control-plane/service-values.yaml b/examples/va/nvidia-mdev/control-plane/service-values.yaml new file mode 100644 index 000000000..e56065b09 --- /dev/null +++ b/examples/va/nvidia-mdev/control-plane/service-values.yaml @@ -0,0 +1,44 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: service-values + annotations: + config.kubernetes.io/local-config: "true" +data: + preserveJobs: false + neutron: + customServiceConfig: | + [DEFAULT] + service_plugins=qos,ovn-router,trunk,segments,port_forwarding,log,placement + [ml2] + mechanism_drivers=sriovnicswitch,ovn + extension_drivers=port_security,router,qos,segments,trunk,placement,port_numa_affinity_policy + [ml2_type_vlan] + network_vlan_ranges=datacentre:1:1000,physnet1:2000:2005 + ovn: + ovnController: + nicMappings: + datacentre: enp6s0 + glance: + customServiceConfig: | + [DEFAULT] + enabled_backends = default_backend:swift + [glance_store] + default_backend = default_backend + [default_backend] + swift_store_create_container_on_put = True + swift_store_auth_version = 3 + swift_store_auth_address = {{ .KeystoneInternalURL }} + swift_store_endpoint_type = internalURL + swift_store_user = service:glance + swift_store_key = {{ .ServicePassword }} + default: + replicas: 1 + swift: + enabled: true + nova: + schedulerServiceTemplate: + customServiceConfig: | + [filter_scheduler] + enabled_filtes = AvailabilityZoneFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,PciPassthroughFilter,NUMATopologyFilter,AggregateInstanceExtraSpecsFilter diff --git a/examples/va/nvidia-mdev/edpm-post-driver/.gitignore b/examples/va/nvidia-mdev/edpm-post-driver/.gitignore new file mode 100644 index 000000000..835442d0a --- /dev/null +++ b/examples/va/nvidia-mdev/edpm-post-driver/.gitignore @@ -0,0 +1,2 @@ +dataplane-deployment.yaml +dataplane-nodeset.yaml \ No newline at end of file diff --git a/examples/va/nvidia-mdev/edpm-post-driver/deployment/.gitignore b/examples/va/nvidia-mdev/edpm-post-driver/deployment/.gitignore new file mode 100644 index 000000000..56387c5df --- /dev/null +++ b/examples/va/nvidia-mdev/edpm-post-driver/deployment/.gitignore @@ -0,0 +1 @@ +dataplane-deployment.yaml \ No newline at end of file diff --git a/examples/va/nvidia-mdev/edpm-post-driver/deployment/kustomization.yaml b/examples/va/nvidia-mdev/edpm-post-driver/deployment/kustomization.yaml new file mode 100644 index 000000000..e2a4f0d48 --- /dev/null +++ b/examples/va/nvidia-mdev/edpm-post-driver/deployment/kustomization.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../../va/nvidia-mdev/edpm-post-driver/deployment + # - https://github.com/openstack-k8s-operators/architecture/va/nvidia-mdev/edpm-post-driver/deployment?ref=main + ## It's possible to replace ../../../../../va/nvidia-mdev/edpm-post-driver/deployment/ with a git checkout URL as per: + ## https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md + +resources: + - values.yaml + +replacements: + - source: + kind: ConfigMap + name: edpm-nodeset-post-driver-values + fieldPath: data.servicesOverride + targets: + - select: + kind: OpenStackDataPlaneDeployment + fieldPaths: + - spec.servicesOverride + options: + create: true + - source: + kind: ConfigMap + name: edpm-nodeset-post-driver-values + fieldPath: data.ansibleExtraVars + targets: + - select: + kind: OpenStackDataPlaneDeployment + fieldPaths: + - spec.ansibleExtraVars + options: + create: true + - source: + kind: ConfigMap + name: edpm-nodeset-post-driver-values + fieldPath: data.deployment.name + targets: + - select: + kind: OpenStackDataPlaneDeployment + fieldPaths: + - metadata.name + options: + create: true diff --git a/examples/va/nvidia-mdev/edpm-post-driver/deployment/values.yaml b/examples/va/nvidia-mdev/edpm-post-driver/deployment/values.yaml new file mode 100644 index 000000000..d00a742ec --- /dev/null +++ b/examples/va/nvidia-mdev/edpm-post-driver/deployment/values.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: ConfigMap + +metadata: + name: edpm-nodeset-post-driver-values + annotations: + config.kubernetes.io/local-config: "true" + +data: + deployment: + name: edpm-deployment-post-driver + servicesOverride: + - reboot-os + ansibleExtraVars: + edpm_reboot_strategy: force diff --git a/examples/va/nvidia-mdev/edpm/nodeset/kustomization.yaml b/examples/va/nvidia-mdev/edpm/nodeset/kustomization.yaml index 31278ef62..e1a68d45b 100644 --- a/examples/va/nvidia-mdev/edpm/nodeset/kustomization.yaml +++ b/examples/va/nvidia-mdev/edpm/nodeset/kustomization.yaml @@ -10,3 +10,65 @@ components: resources: - values.yaml + +replacements: + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.root_password + targets: + - select: + kind: Secret + name: baremetalset-password-secret + fieldPaths: + - data.NodeRootPassword + options: + create: true + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.preProvisioned + targets: + - select: + kind: OpenStackDataPlaneNodeSet + name: openstack-edpm + fieldPaths: + - spec.preProvisioned + options: + create: true + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.baremetalSetTemplate + targets: + - select: + kind: OpenStackDataPlaneNodeSet + name: openstack-edpm + fieldPaths: + - spec.baremetalSetTemplate + options: + create: true + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.nova.compute.conf + targets: + - select: + kind: ConfigMap + name: cpu-pinning-nova + fieldPaths: + - data.25-cpu-pinning-nova\.conf + options: + create: true + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.nova.pci.conf + targets: + - select: + kind: ConfigMap + name: sriov-nova + fieldPaths: + - data.03-sriov-nova\.conf + options: + create: true diff --git a/examples/va/nvidia-mdev/edpm/nodeset/values.yaml b/examples/va/nvidia-mdev/edpm/nodeset/values.yaml index d7a2f949a..6a04abc8a 100644 --- a/examples/va/nvidia-mdev/edpm/nodeset/values.yaml +++ b/examples/va/nvidia-mdev/edpm/nodeset/values.yaml @@ -122,20 +122,20 @@ data: services: - bootstrap - download-cache + - reboot-os - configure-network - validate-network - install-os - configure-os - ssh-known-hosts - run-os - - reboot-os - install-certs - - libvirt - ovn - neutron-ovn - - nova-custom-sriov - - neutron-sriov - neutron-metadata + - neutron-sriov + - libvirt + - nova-custom-sriov nova: compute: conf: | diff --git a/examples/va/nvidia-mdev/kustomization.yaml b/examples/va/nvidia-mdev/kustomization.yaml deleted file mode 100644 index 7fb587ab3..000000000 --- a/examples/va/nvidia-mdev/kustomization.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -components: - - ../../../va/nvidia-mdev/ - # - https://github.com/openstack-k8s-operators/architecture/va/nvidia-mdev?ref=main - ## It's possible to replace ../../../va/nvidia-mdev/ with a git checkout URL as per: - ## https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md - -resources: - - nncp/values.yaml - - service-values.yaml diff --git a/examples/va/nvidia-mdev/service-values.yaml b/examples/va/nvidia-mdev/service-values.yaml deleted file mode 100644 index 7d79a5db9..000000000 --- a/examples/va/nvidia-mdev/service-values.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: service-values - annotations: - config.kubernetes.io/local-config: "true" -data: - preserveJobs: false diff --git a/va/nvidia-mdev/edpm-post-driver/deployment/kustomization.yaml b/va/nvidia-mdev/edpm-post-driver/deployment/kustomization.yaml new file mode 100644 index 000000000..227bcc100 --- /dev/null +++ b/va/nvidia-mdev/edpm-post-driver/deployment/kustomization.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +transformers: + # Set namespace to OpenStack on all namespaced objects without a namespace + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../../../lib/dataplane/deployment diff --git a/va/nvidia-mdev/edpm/nodeset/kustomization.yaml b/va/nvidia-mdev/edpm/nodeset/kustomization.yaml index 0c4325864..0bea6a258 100644 --- a/va/nvidia-mdev/edpm/nodeset/kustomization.yaml +++ b/va/nvidia-mdev/edpm/nodeset/kustomization.yaml @@ -37,54 +37,51 @@ replacements: - data.NodeRootPassword options: create: true - - # Nova compute CPU pinning customization - source: kind: ConfigMap name: edpm-nodeset-values - fieldPath: data.nova.compute.conf + fieldPath: data.preProvisioned targets: - select: - kind: ConfigMap - name: cpu-pinning-nova + kind: OpenStackDataPlaneNodeSet + name: openstack-edpm fieldPaths: - - data.25-cpu-pinning-nova\.conf + - spec.preProvisioned options: create: true - # Nova compute PCI passthrough customization - source: kind: ConfigMap name: edpm-nodeset-values - fieldPath: data.nova.pci.conf + fieldPath: data.baremetalSetTemplate targets: - select: - kind: ConfigMap - name: sriov-nova + kind: OpenStackDataPlaneNodeSet + name: openstack-edpm fieldPaths: - - data.03-sriov-nova\.conf + - spec.baremetalSetTemplate options: create: true - source: kind: ConfigMap name: edpm-nodeset-values - fieldPath: data.preProvisioned + fieldPath: data.nova.compute.conf targets: - select: - kind: OpenStackDataPlaneNodeSet - name: openstack-edpm + kind: ConfigMap + name: cpu-pinning-nova fieldPaths: - - spec.preProvisioned + - data.25-cpu-pinning-nova\.conf options: create: true - source: kind: ConfigMap name: edpm-nodeset-values - fieldPath: data.baremetalSetTemplate + fieldPath: data.nova.pci.conf targets: - select: - kind: OpenStackDataPlaneNodeSet - name: openstack-edpm + kind: ConfigMap + name: sriov-nova fieldPaths: - - spec.baremetalSetTemplate + - data.03-sriov-nova\.conf options: create: true