Skip to content

Commit

Permalink
[Draft] Split nvidia-mdev arch in two stages
Browse files Browse the repository at this point in the history
  • Loading branch information
sbauza authored and jamepark4 committed Dec 11, 2024
1 parent 2944874 commit 5a42edd
Show file tree
Hide file tree
Showing 16 changed files with 335 additions and 54 deletions.
31 changes: 23 additions & 8 deletions automation/vars/nvidia-mdev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
vas:
nvidia-mdev:
stages:
- path: examples/va/nvidia-mdev/nncp
- path: examples/va/nvidia-mdev/control-plane/nncp
wait_conditions:
- >-
oc -n openstack wait nncp
Expand All @@ -14,7 +14,7 @@ vas:
src_file: values.yaml
build_output: nncp.yaml

- path: examples/va/nvidia-mdev
- path: examples/va/nvidia-mdev/control-plane
wait_conditions:
- >-
oc -n openstack wait osctlplane controlplane --for condition=Ready
Expand All @@ -36,12 +36,6 @@ vas:
- name: edpm-nodeset-values
src_file: values.yaml
build_output: nodeset.yaml
post_stage_run:
- name: Install nvidia driver
type: playbook
# As a reminder, the job needs to set the nvidia driver URL
source: "../../playbooks/nvidia-mdev.yml"
inventory: "${HOME}/ci-framework-data/artifacts/zuul_inventory.yml"

- path: examples/va/nvidia-mdev/edpm/deployment
wait_conditions:
Expand All @@ -53,3 +47,24 @@ vas:
- name: edpm-deployment-values
src_file: values.yaml
build_output: deployment.yaml
#post_stage_run:

Check warning on line 50 in automation/vars/nvidia-mdev.yaml

View workflow job for this annotation

GitHub Actions / yaml-lint

50:10 [comments] missing starting space in comment
#- name: Run phase 1 playbook

Check warning on line 51 in automation/vars/nvidia-mdev.yaml

View workflow job for this annotation

GitHub Actions / yaml-lint

51:12 [comments] missing starting space in comment

Check warning on line 51 in automation/vars/nvidia-mdev.yaml

View workflow job for this annotation

GitHub Actions / yaml-lint

51:11 [comments-indentation] comment not indented like content
# type: playbook
# # As a reminder, the job needs to set the nvidia driver URL
# source: "../../playbooks/nvidia-mdev-phase1.yml"
# inventory: "${HOME}/ci-framework-data/artifacts/zuul_inventory.yml"
#- name: Run phase 2 playbook

Check warning on line 56 in automation/vars/nvidia-mdev.yaml

View workflow job for this annotation

GitHub Actions / yaml-lint

56:12 [comments] missing starting space in comment
# type: playbook
# source: "../../playbooks/nvidia-mdev-phase2.yml"
# inventory: "${HOME}/ci-framework-data/artifacts/zuul_inventory.yml"

- path: examples/va/nvidia-mdev/edpm-post-driver/deployment
wait_conditions:
- >-
oc -n openstack wait
osdpd edpm-deployment-post-driver --for condition=Ready
--timeout=60m
values:
- name: edpm-post-driver-deployment-values
src_file: values.yaml
build_output: deployment-post-driver.yaml
98 changes: 98 additions & 0 deletions examples/va/nvidia-mdev/control-plane/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
---
apiVersion: kustomize.config.k8s.io/v1alpha1
kind: Component

transformers:
# Set namespace to OpenStack on all namespaced objects without a namespace
- |-
apiVersion: builtin
kind: NamespaceTransformer
metadata:
name: _ignored_
namespace: openstack
setRoleBindingSubjects: none
unsetOnly: true
fieldSpecs:
- path: metadata/name
kind: Namespace
create: true
components:
- ../../../../lib/networking/metallb
- ../../../../lib/networking/netconfig
- ../../../../lib/networking/nad
- ../../../../lib/control-plane

resources:
- nncp/values.yaml
- service-values.yaml

replacements:
# Control plane customization
- source:
kind: ConfigMap
name: service-values
fieldPath: data.neutron.customServiceConfig
targets:
- select:
kind: OpenStackControlPlane
fieldPaths:
- spec.neutron.template.customServiceConfig
options:
create: true
# OVN control plane SRIOV customization
- source:
kind: ConfigMap
name: service-values
fieldPath: data.ovn.ovnController.nicMappings
targets:
- select:
kind: OpenStackControlPlane
fieldPaths:
- spec.ovn.template.ovnController.nicMappings
options:
create: true
- source:
kind: ConfigMap
name: service-values
fieldPath: data.glance.customServiceConfig
targets:
- select:
kind: OpenStackControlPlane
fieldPaths:
- spec.glance.template.customServiceConfig
options:
create: true
- source:
kind: ConfigMap
name: service-values
fieldPath: data.glance.default.replicas
targets:
- select:
kind: OpenStackControlPlane
fieldPaths:
- spec.glance.template.glanceAPIs.default.replicas
options:
create: true
- source:
kind: ConfigMap
name: service-values
fieldPath: data.swift.enabled
targets:
- select:
kind: OpenStackControlPlane
fieldPaths:
- spec.swift.enabled
options:
create: true
- source:
kind: ConfigMap
name: service-values
fieldPath: data.nova.schedulerServiceTemplate.customServiceConfig
targets:
- select:
kind: OpenStackControlPlane
fieldPaths:
- spec.nova.template.schedulerServiceTemplate.customServiceConfig
options:
create: true
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ transformers:
create: true
components:
- ../../../../lib/nncp
- ../../../../../lib/nncp

resources:
- values.yaml
File renamed without changes.
44 changes: 44 additions & 0 deletions examples/va/nvidia-mdev/control-plane/service-values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: service-values
annotations:
config.kubernetes.io/local-config: "true"
data:
preserveJobs: false
neutron:
customServiceConfig: |
[DEFAULT]
service_plugins=qos,ovn-router,trunk,segments,port_forwarding,log,placement
[ml2]
mechanism_drivers=sriovnicswitch,ovn
extension_drivers=port_security,router,qos,segments,trunk,placement,port_numa_affinity_policy
[ml2_type_vlan]
network_vlan_ranges=datacentre:1:1000,physnet1:2000:2005
ovn:
ovnController:
nicMappings:
datacentre: enp6s0
glance:
customServiceConfig: |
[DEFAULT]
enabled_backends = default_backend:swift
[glance_store]
default_backend = default_backend
[default_backend]
swift_store_create_container_on_put = True
swift_store_auth_version = 3
swift_store_auth_address = {{ .KeystoneInternalURL }}
swift_store_endpoint_type = internalURL
swift_store_user = service:glance
swift_store_key = {{ .ServicePassword }}
default:
replicas: 1
swift:
enabled: true
nova:
schedulerServiceTemplate:
customServiceConfig: |
[filter_scheduler]
enabled_filtes = AvailabilityZoneFilter,ComputeFilter,ComputeCapabilitiesFilter,ImagePropertiesFilter,ServerGroupAntiAffinityFilter,ServerGroupAffinityFilter,PciPassthroughFilter,NUMATopologyFilter,AggregateInstanceExtraSpecsFilter
2 changes: 2 additions & 0 deletions examples/va/nvidia-mdev/edpm-post-driver/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
dataplane-deployment.yaml
dataplane-nodeset.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
dataplane-deployment.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

components:
- ../../../../../va/nvidia-mdev/edpm-post-driver/deployment
# - https://github.com/openstack-k8s-operators/architecture/va/nvidia-mdev/edpm-post-driver/deployment?ref=main
## It's possible to replace ../../../../../va/nvidia-mdev/edpm-post-driver/deployment/ with a git checkout URL as per:
## https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md

resources:
- values.yaml

replacements:
- source:
kind: ConfigMap
name: edpm-nodeset-post-driver-values
fieldPath: data.servicesOverride
targets:
- select:
kind: OpenStackDataPlaneDeployment
fieldPaths:
- spec.servicesOverride
options:
create: true
- source:
kind: ConfigMap
name: edpm-nodeset-post-driver-values
fieldPath: data.ansibleExtraVars
targets:
- select:
kind: OpenStackDataPlaneDeployment
fieldPaths:
- spec.ansibleExtraVars
options:
create: true
- source:
kind: ConfigMap
name: edpm-nodeset-post-driver-values
fieldPath: data.deployment.name
targets:
- select:
kind: OpenStackDataPlaneDeployment
fieldPaths:
- metadata.name
options:
create: true
16 changes: 16 additions & 0 deletions examples/va/nvidia-mdev/edpm-post-driver/deployment/values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
---
apiVersion: v1
kind: ConfigMap

metadata:
name: edpm-nodeset-post-driver-values
annotations:
config.kubernetes.io/local-config: "true"

data:
deployment:
name: edpm-deployment-post-driver
servicesOverride:
- reboot-os
ansibleExtraVars:
edpm_reboot_strategy: force
62 changes: 62 additions & 0 deletions examples/va/nvidia-mdev/edpm/nodeset/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,65 @@ components:

resources:
- values.yaml

replacements:
- source:
kind: ConfigMap
name: edpm-nodeset-values
fieldPath: data.root_password
targets:
- select:
kind: Secret
name: baremetalset-password-secret
fieldPaths:
- data.NodeRootPassword
options:
create: true
- source:
kind: ConfigMap
name: edpm-nodeset-values
fieldPath: data.preProvisioned
targets:
- select:
kind: OpenStackDataPlaneNodeSet
name: openstack-edpm
fieldPaths:
- spec.preProvisioned
options:
create: true
- source:
kind: ConfigMap
name: edpm-nodeset-values
fieldPath: data.baremetalSetTemplate
targets:
- select:
kind: OpenStackDataPlaneNodeSet
name: openstack-edpm
fieldPaths:
- spec.baremetalSetTemplate
options:
create: true
- source:
kind: ConfigMap
name: edpm-nodeset-values
fieldPath: data.nova.compute.conf
targets:
- select:
kind: ConfigMap
name: cpu-pinning-nova
fieldPaths:
- data.25-cpu-pinning-nova\.conf
options:
create: true
- source:
kind: ConfigMap
name: edpm-nodeset-values
fieldPath: data.nova.pci.conf
targets:
- select:
kind: ConfigMap
name: sriov-nova
fieldPaths:
- data.03-sriov-nova\.conf
options:
create: true
8 changes: 4 additions & 4 deletions examples/va/nvidia-mdev/edpm/nodeset/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -122,20 +122,20 @@ data:
services:
- bootstrap
- download-cache
- reboot-os
- configure-network
- validate-network
- install-os
- configure-os
- ssh-known-hosts
- run-os
- reboot-os
- install-certs
- libvirt
- ovn
- neutron-ovn
- nova-custom-sriov
- neutron-sriov
- neutron-metadata
- neutron-sriov
- libvirt
- nova-custom-sriov
nova:
compute:
conf: |
Expand Down
13 changes: 0 additions & 13 deletions examples/va/nvidia-mdev/kustomization.yaml

This file was deleted.

9 changes: 0 additions & 9 deletions examples/va/nvidia-mdev/service-values.yaml

This file was deleted.

Loading

0 comments on commit 5a42edd

Please sign in to comment.