Merge more-images and StackHPC changes to main

.gitignore

@@ -109,8 +109,19 @@ ansible/*.retry
 ansible/roles/*\.*/
 # Galaxy collections
 ansible/collections/**/
+# Image cache
+ansible/openstack-config-image-cache
+# Venvs
+ansible/openstack-config-venv
 
 # Editors
 *~
 .*.swp
 .*sw?
+
+# Ignore working dirs
+ansible/openstack-config-image-cache
+ansible/openstack-config-venv
+
+# Ignore tmp output from template generation playbook
+generated-magnum-snippets/

README.rst

@@ -17,17 +17,12 @@ packages. For example:
 
    $ virtualenv venv
    $ source venv/bin/activate
-   $ pip install -U pip
+   $ python -m pip install --upgrade pip
    $ pip install -r requirements.txt
 
 Install Ansible role and collection dependencies from Ansible Galaxy:
 
 .. code-block::
-
-   $ ansible-galaxy role install \
-        -p ansible/roles \
-        -r requirements.yml
-
    $ ansible-galaxy collection install \
        -p ansible/collections \
        -r requirements.yml
@@ -81,3 +76,33 @@ configuration parameter:
 .. code-block::
 
    $ tools/openstack-config -- --vault-password-file config-secret.vault
+
+
+Magnum Cluster Templates
+========================
+
+To generate a new set of Magnum cluster templates and corresponding Glance image
+definitions which utilise the latest stable upstream release tag, set the following
+variables in `etc/openstack-config.yml`
+
+.. code-block:: yaml
+
+   magnum_default_master_flavor_name: # Chosen flavor on target cloud
+   magnum_default_worker_flavor_name: # Chosen flavor on target cloud
+   magnum_external_net_name: # External network
+   magnum_loadbalancer_provider: # Octavia provider (e.g. 'ovn')
+
+then run the provided playbook with
+
+.. code-block:: bash
+
+   $ tools/openstack-config -p ansible/generate-magnum-capi-templates.yml
+
+This will create a ``generated-magnum-snippets`` directory in the repo root with
+a timestamped sub-directory containing an ``images.yml`` file and a ``templates.yml``
+file. The contents of these two files can then be added to any existing images and
+cluster templates in ``etc/openstack-config.yml``. When deploying the updated config,
+be sure to run the ``openstack-images.yml`` playbook *before* running the
+``openstack-container-clusters.yml`` playbook, otherwise the Magnum API will return
+an error referencing an invalid cluster type with image ``None``. This is handled
+automatically if running the full ``openstack.yml`` playbook.

ansible/generate-magnum-capi-templates.yml

@@ -0,0 +1,60 @@
+---
+- name: Generate cluster templates
+  hosts: localhost
+  vars:
+    root_dir: ../
+  tasks:
+
+  - name: Check that required variables are defined
+    assert:
+      that:
+        - magnum_default_master_flavor_name is defined
+        - magnum_default_worker_flavor_name is defined
+        - magnum_external_net_name is defined
+        - magnum_loadbalancer_provider is defined
+
+  - name: Fetch capi-helm-charts release information
+    ansible.builtin.uri:
+      url: https://api.github.com/repos/stackhpc/capi-helm-charts/releases/latest
+    register: capi_helm_chart_release_data
+
+  - name: Fetch dependencies.json for capi-helm-charts release
+    ansible.builtin.uri:
+      url: https://raw.githubusercontent.com/stackhpc/capi-helm-charts/{{ capi_helm_chart_release_data.json.tag_name }}/dependencies.json
+    register: dependencies_response
+
+  - name: Ensure wget packages is installed
+    become: true
+    package:
+      name: wget
+      state: present
+
+  - name: Fetch manifest.json for capi-helm-charts images
+    # ansible.builtin.uri:
+    #   url: https://raw.githubusercontent.com/stackhpc/azimuth-images/{{ dependencies_response.json['azimuth-images'] }}/manifest.json
+    # Above URL returns 404 even though similar URL for capi-helm-charts repo works fine
+    # Not sure why but fall back to wget + JSON parsing for now.
+    shell: "wget -O - https://github.com/stackhpc/azimuth-images/releases/download/{{ dependencies_response.json['azimuth-images'] }}/manifest.json"
+    register: manifest_response
+    changed_when: false
+
+  - name: Parse JSON response
+    set_fact:
+      new_template_data: "{{ manifest_response.stdout | from_json | dict2items | selectattr('key', 'match', 'kubernetes*') | list }}"
+
+  - name: Ensure output dir exists
+    ansible.builtin.file:
+      path: "{{ [root_dir, 'generated-magnum-snippets', now(utc=true,fmt='%Y-%m-%d-T%H-%M-%S')] | path_join }}"
+      state: directory
+      mode: '0755'
+    register: output_dir
+
+  - name: Write new image config to file
+    template:
+      src: "magnum-capi-images.j2"
+      dest: "{{ output_dir.path }}/images.yml"
+
+  - name: Write new cluster template config to file
+    template:
+      src: "magnum-capi-templates.j2"
+      dest: "{{ output_dir.path }}/templates.yml"

ansible/group_vars/all/openstack

@@ -2,6 +2,10 @@
 ###############################################################################
 # Configuration of OpenStack user environment for OpenStack.
 
+# List of OpenStack domains. Format is as required by the stackhpc.os-projects
+# role.
+openstack_domains: []
+
 # List of OpenStack projects. Format is as required by the stackhpc.os-projects
 # role.
 openstack_projects: []
@@ -21,6 +25,10 @@ openstack_routers: []
 # stackhpc.os-networks role.
 openstack_security_groups: []
 
+# List of RBAC definitions in the openstack projct. Format is as required by the
+# stackhpc.os-networks role.
+openstack_networks_rbac: []
+
 ###############################################################################
 # Configuration of nova flavors for OpenStack.
 

ansible/openstack-container-clusters.yml

@@ -4,7 +4,7 @@
   tags:
     - container-clusters-templates
   roles:
-    - role: stackhpc.os-container-clusters
+    - role: stackhpc.openstack.os_container_clusters
       os_container_clusters_venv: "{{ openstack_venv }}"
       os_container_clusters_auth_type: "{{ openstack_auth_type }}"
       os_container_clusters_auth: "{{ openstack_auth }}"

ansible/openstack-flavors.yml

@@ -4,7 +4,7 @@
   tags:
     - flavors
   roles:
-    - role: stackhpc.os-flavors
+    - role: stackhpc.openstack.os_flavors
       os_flavors_venv: "{{ openstack_venv }}"
       os_flavors_auth_type: "{{ openstack_auth_type }}"
       os_flavors_auth: "{{ openstack_auth }}"

ansible/openstack-host-aggregates.yml

@@ -4,7 +4,7 @@
   tags:
     - host_aggregates
   roles:
-    - role: stackhpc.os_host_aggregates
+    - role: stackhpc.openstack.os_host_aggregates
       os_host_aggregates_venv: "{{ openstack_venv }}"
       os_host_aggregates_auth_type: "{{ openstack_auth_type }}"
       os_host_aggregates_auth: "{{ openstack_auth }}"

ansible/openstack-images.yml

@@ -4,7 +4,7 @@
   tags:
     - images
   roles:
-    - role: stackhpc.os-images
+    - role: stackhpc.openstack.os_images
       os_images_venv: "{{ openstack_venv }}"
       os_images_cache: "{{ ansible_env.PWD }}/openstack-config-image-cache"
       os_images_auth_type: "{{ openstack_auth_type }}"

ansible/openstack-networks.yml

@@ -4,11 +4,12 @@
   tags:
     - networks
   roles:
-    - role: stackhpc.os-networks
+    - role: stackhpc.openstack.os_networks
       os_networks_venv: "{{ openstack_venv }}"
       os_networks_auth_type: "{{ openstack_auth_type }}"
       os_networks_auth: "{{ openstack_auth }}"
       os_networks_cacert: "{{ openstack_cacert }}"
       os_networks: "{{ openstack_networks }}"
       os_networks_routers: "{{ openstack_routers }}"
       os_networks_security_groups: "{{ openstack_security_groups }}"
+      os_networks_rbac: "{{ openstack_networks_rbac }}"

ansible/openstack-project.yml

@@ -4,9 +4,10 @@
   tags:
     - project
   roles:
-    - role: stackhpc.os-projects
+    - role: stackhpc.openstack.os_projects
       os_projects_venv: "{{ openstack_venv }}"
       os_projects_auth_type: "{{ openstack_auth_type }}"
       os_projects_admin_auth: "{{ openstack_auth }}"
       os_projects_cacert: "{{ openstack_cacert }}"
       os_projects: "{{ openstack_projects }}"
+      os_projects_domains: "{{ openstack_domains }}"

ansible/secrets/nvidia-client-token.tok

@@ -0,0 +1,140 @@
+$ANSIBLE_VAULT;1.1;AES256
+37366431656638653236363833623031663536646163326536386265303135323237353665653638
+3634653939306336373662656461663832653866373563320a373336393064366363386263333764
+61313162303263383936373437653334306565373232613132343335356533656361386166326335
+3235636135376136360a356636626338613230316233336538616236303538313237653934613132
+65623064386539663866663432376430383565376231393939646435636264326463653033313038
+62333365396436303737626638333833353264343965376538346632623966363131646639646561
+32333161313464316533613230636131613336303863303131326631636163623631326166323438
+66636232376461303231343930303535303532663361663034336562366537353863666137363631
+63633133373639376338633234613766326433366361636637393638613362326434623337333939
+66316564316639383534643465623837633039333763613864336165363566323733343064663930
+38353965323763303934393465303731306332613935646636633233393531323033353031653063
+64643663653665343937363439373835383033373365633637306162356464663066323830633636
+38306265386562396137336431633331396137643338313061313061393061306238346634323138
+64306437343865386261373561656661333062323963326334663764613836346266393535386463
+61383031356665363233313637343533366539633734336365643834646438653265383862666336
+63353163623635386233356462636232316563303136383764616633336432316633376266326365
+61393564333532323836383564373537383435396464366536393130666236356536393634333436
+35643765396663373035306166376530663530366233373331646533333038376435653964636236
+63353531356663383234636238383239643462646366363439373036396238353132333537623237
+63346538333637373131613332383237663933643034653437323337636338653434646131386136
+65646538326466653733646666343266343737393138323038653161393163623433363632633633
+38633632323033396539366363316239636233656565653334336266383531393231386239643233
+63623939306366333131373739633434373733363864386438653938306562376131626665663433
+34623365313838353762393361386136303932653631363661336533343263636333393465636330
+33396239636232393535356233623735656665643530643963306236326235323631363362383033
+37383661343535303561363066396236353463613437643966393762346466373562343163653661
+61663437353932666239656263363739643135666138356634393335326336333739343733326133
+35373332663438626264306432643037323763386261396363343065343866393731623136346432
+61356331353662306633643563313964386637643938336234346636323834623562373431623831
+65336561653161663665303537656336323339336262303833383838376537623030343633626130
+39643838326634313934623139663064383634633338386261343931343464393733636630633033
+30383937666364663333396366346335313565373337636362306262343861323061366433303232
+65303030353934313931393466633432363761623837656332356333336561623039633938623066
+64343238373134313735636561613866313761376463333731336361613865663261656535346461
+61613534663566313236646364616231656664343138383236326265636235616136323130373864
+32613134663431613034383435326431306136656638653164633439313865346464383663663265
+38373230366361363866383330633836353535393039616231636662663437353962613863646634
+63356637366164313131333231393466383133363863306164613030393566366534613063653636
+38393562616330383136333932376330366562366238356266323063313030386432333966353236
+34663934343231643630396331306231616561363833383034353062656537353363376463383162
+34636538643233383137663733303638613961323735363636646630373337383636643837343534
+37643035356436656430333038663938646662363837346238303735353539333661346239386239
+30336565323166663564373034613430323931633562363337316533383066616532313163393439
+32656162343362633038356330366233353133303135613266663935653338346139333135613964
+33666438636166303265303937623738663439386630373735626433653738613334346261353131
+36323864633665306133643332396231366332666461356331306237373163613236643664366364
+33653433636262326232323136343636383362373638323530323466306161363338343364653735
+64393436313238616331333438396539386666663532396233396462363266323265383537343532
+35663838313065356630336361663761366532663934633564333436376537346637393238616535
+34333564646230353336633037373038393262646361396332353161376563333631663563643261
+62323633386431643766623361613866656232316232386132326538383635636538613639326635
+65316665313433303031313939623831333930363464323737373832386632363139346664653033
+36373638386435326236623866326636623531306364353737393634386662353136343033636664
+37616265343664396534656633626634313761373435366666636238653030626564616165303864
+36353964373562383832626230653139643934613938353432613335623966383364623233666532
+36366363336533353561363630376162643635653863623064643634303061313863376463626162
+32326138613432373132376366313163656339636261343133393233363033613335316262643963
+66646533346230383936383963636532636666663265343230316263626666366365353965346239
+65373964366432353233313232663765343765393963373163333331386364316166666537353431
+65663339633139393532613430663437363865376637633835663464316231303939373331323839
+38336435613433353339336162633335353134393966323931353634663865613539383764343032
+38303036633639383733633963643138396463643932656538376534396562626432363138636533
+61366362656236393536643139333964336265326539323334306462633965363966616432663337
+33643730363532313439396536326338313539313732623334353332313963363631366663663164
+32363033336565356363633661636538356437663538346237356636616331613266653534326363
+64653862613763313830373532633437653737396265636464353161323939373530616239393632
+35646631633263333537393931393232383433396236313065306437653731313066316631333735
+64623132303638353334336463646138306362656139633731356636643737333737303738376635
+32646639366165353866633466353337666236383434623133653736303633663934346132303736
+39343966653665356231646439613334656237623663393065373366363362356336356538323966
+66623966356461393231623732356534646564643763626161303430626561633339633639353831
+30303332383565646237333965653339643539313062323364353335343738663333333061663535
+31393835333234353736393039313530336332303363653763386438376663646535363065373632
+33666363376137623561646231653137363366333332626164653534613766323338393932613533
+64333937633839626235366465623262313063393132653933656539353637623766336531323939
+61323966613564386330636261383961383330623834316331613064656332356134393536656461
+38316633666263626133386437613865633436356432663261336335323834363433303462656633
+31613034303733613663363333313937353931356262343937303136616666616637303165346566
+65333766326663356664323731356264393737353430633066346132353665343361616235393939
+35356434383662626664313132636539376562393238323462306532326639386566616236623665
+30653734303333346661363132376664366264643830376635303838643738366230363731333664
+61333362326134386464373030633634373138363461376133333764313532323364663838336664
+38353161306233633731343863636139396138653761623931346533653937316531316230646162
+62663430663339366463343032343366376436313936363133383237646534336361613534363237
+36626236376536626361666137356263366236613935363231363366306563326366613663313338
+62653861653264643136306565303064373166626163646466303733363837353434613463396430
+37633031383261363435646233663435336165643238663263646165623961666463323333373138
+32613032343637363330646432323265306139396637323531396330623030306437393435323533
+66653638343636353734373233633065333562366361303137326334666532303636333431356264
+37636435363462323838336330363934643330393336373864643531666236326536353265343564
+61323831623562393437333264383664383461333965316536326365353761663632643238643935
+65313933393664386566363161633364633562383134396131643236663834613661323136386533
+64306561656532666162346232303262353135633434383834393835636333306266633630313835
+64626335396661616335303465336631383366366463353638373131386636636436363438636636
+35613834373932376665323066633561333739383736313431346665353335663462316463376137
+64313837363533626538396332383266373665393633373835626234356339326564343962626435
+62623730303839366637623534323833636334616534653132663839383762646361613563646366
+62343034653037303030306132376664653536373865393831653733396436653465336137396566
+32373565653533333336363662373939626633663965623866623730636665656432376330616562
+63333338346263393761343935623037393735316363333233366637386263633238366464633361
+31306265393365306630656239323462346562646432336566336364306130313334613861363331
+31363631366436333034663739353038643333633732336563633536306131326136646237376630
+66343032653631366363636534663363613863366563353136383830313037616232366335333461
+30323730323635633736663264343736353639616234356566393664643561366436363636616461
+62333830346530616636373034363338346330653735323065393236343432366263353939383132
+65386266363537346139623836613563633935643030376263336634646333336564633563323639
+35643130303466636136623037303036633465343537626131323432653937643766333666363238
+65383338346431383236613364666331363937303836396639306261336232633433373731313163
+32316361613239623137316464373033323661376336336233306130666635643839336232373637
+36363464643830663834363933613061623764353933626166633436666639336239613736383861
+65326436323232643362303932616137373634663034663665356166356634666564636133663038
+34373639653365333031353732303633653262306133303766666561363330653763366438343161
+35316361303666383337316638383134646461663931326265323663636237333563633961383664
+62663337643433643232393532303365353032306535356437643531653235633534626230363762
+63376536623630356333316464306539666132323661383962656461353162326533323764343434
+61633861613364373532383839333632323831393866643162623333316530633335623733633038
+39353032316434336263336331366431333036626334303566313561303236653337623764366439
+34373734643331633830316230356664616239633232326565626235363034643338653531323336
+65643134613635383863366463326438356262393637656235316332353137366436363066386165
+32643636613163356133373065396134373732643737323135303238613334626339393436666438
+61616164633561666466333665346262653965343830366465363333353366316630656661343261
+36393633353630306632616162343762323566376338303464636365653661653061393561303063
+33373739626139376634373765613236366632386532393364643231643838313261633864323836
+39323536666231613766306133393334396261623937353536326533373730313438643137636461
+38383531383066396436373162336466326535393334616438313731383161346439373631336266
+65376632373838656165393262363834303937396635323463656361623937666239663432383538
+39353666626230626135373466613733383130383063326434316539373231343934653266646362
+30376235386130393538396465306537343539396337653731623739373434663864666562666634
+66636532626133303034656661376237363130653631313064656231323637663232646637376564
+32633564316462386138626430323438386532623337386662356363613135373663313830373330
+61313061653266633636313934616631646334316266303735316364323366333631313035666666
+38346663396132383865376564633064653163636361666636333138343731306131333263366331
+38653866626565303936363966316230643032663161623466643231653833396435623461303539
+34373133333461383963656332633666636536303038346138663236353461346665356563616365
+61663436303863373037636334333936323365306133316236616463323031613337343437356231
+65333832633165306539643932653231303537363738663330383962383334343638363034633536
+37383162333166643231366434646364333763626336646335643962333534613730666266393233
+31646637336637643733333764653735396265313237326436646338356232613135613338616561
+636634393539646639373534393432363562

ansible/templates/magnum-capi-images.j2

@@ -0,0 +1,24 @@
+# Images required for corresponding Magnum cluster template
+# To make use of the generated config snippets, copy them to
+# etc/openstack-config and add the images to the openstack_images
+# list.
+
+# List snippet to add to existing openstack_images:
+{% for item in new_template_data %}
+# -{% raw %} "{{ {% endraw %}{{ item.value.name |  replace('-', '_') | replace('.', '_') }}{% raw %} }}"{% endraw %}
+
+{% endfor %}
+
+{% for item in new_template_data %}
+# Image for {{ item.key }}
+{{ item.value.name |  replace('-', '_') | replace('.', '_') }}:
+  name: "{{ item.value.name }}"
+  type: qcow2
+  image_url: "{{ item.value.url }}"
+  visibility: "community"
+  properties:
+    os_distro: "ubuntu"
+    os_version: "20.04"
+    kube_version: "{{ item.value.kubernetes_version }}"
+
+{% endfor %}