Merge branch 'main' into remove_obsolete_vms

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+ignore = 
+    E203,
+    W503
+docstring-convention = google

.git-blame-ignore-revs

@@ -0,0 +1,4 @@
+# Run isort
+537adb7be62ff72a49d255d48db534620d7ea36a
+# Run black
+4bc255289b785d9099e76b0fce600464a305894a

.github/workflows/ci.yml

@@ -4,50 +4,48 @@ name: CI
   pull_request:
   push:
     branches:
-      - master
+      - main
 
 defaults:
   run:
     working-directory: 'vgcn-infrastructure'
 
 jobs:
-  lint:
-    name: Lint
+  yamllint:
+    name: yamllint
     runs-on: ubuntu-latest
     steps:
       - name: Check out the codebase.
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           path: 'vgcn-infrastructure'
 
       - name: Set up Python 3.
         uses: actions/setup-python@v2
         with:
-          python-version: '3.x'
+          python-version: '3.9'
 
-      - name: Install test dependencies.
-        run: pip3 install yamllint
+      - name: Install requirements.
+        run: pip install -r requirements.txt
 
       - name: Lint code.
-        run: |
-          yamllint .
-  validate:
-    name: Validate
+        run: yamllint .
+  schema:
+    name: Validate resource definition
     runs-on: ubuntu-latest
     steps:
       - name: Check out the codebase.
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           path: 'vgcn-infrastructure'
 
       - name: Set up Python 3.
         uses: actions/setup-python@v2
         with:
-          python-version: '3.x'
+          python-version: '3.9'
 
-      - name: Install test dependencies.
+      - name: Install requirements.
         run: pip install -r requirements.txt
 
-      - name: Validate code.
-        run: |
-          pykwalify -d resources.yaml -s schema.yaml
+      - name: Validate resource definition.
+        run: pykwalify -d resources.yaml -s schema.yaml

.gitignore

@@ -1,4 +1,3 @@
-
 # Created by https://www.gitignore.io/api/python
 
 ### Python ###
@@ -105,3 +104,7 @@ venv.bak/
 .mypy_cache/
 
 # End of https://www.gitignore.io/api/python
+
+.vault_password
+
+.idea

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "mounts"]
+	path = mounts
+	url = https://github.com/usegalaxy-eu/mounts

README.md

@@ -1,13 +1,26 @@
-# VGCN Infrastructure Management
+# VGCN infrastructure management
 
-This repository allows for definition and management of VGNC resources in the
-bwCloud for usegalaxy.eu
+This repository defines and manages the
+[Virtual Galaxy Compute Nodes](https://github.com/usegalaxy-eu/vgcn) on
+[bwCloud](https://www.bw-cloud.org/)/[de.NBI-cloud](https://www.denbi.de/)
+for [usegalaxy.eu](https://usegalaxy.eu/)
+
+The compute nodes are defined in [`resources.yaml`](#resourcesyaml), which 
+conforms to [`schema.yaml`](schema.yaml).
+[`userdata.yaml.j2`](userdata.yaml.j2) contains actions that will be run by
+[cloud-init](https://cloudinit.readthedocs.io/en/latest/index.html) during the
+first boot of the virtual machines (see 
+[cloud-init docs](https://cloudinit.readthedocs.io/en/23.2.1/explanation/format.html)).
+
+The Jenkins project 
+[vgcn-infrastructure](https://build.galaxyproject.eu/job/usegalaxy-eu/job/vgcn-infrastructure/)
+runs [`synchronize.py`](synchronize.py) periodically to deploy the configured
+compute nodes.
 
 ## `resources.yaml`
 
-This file defines resources that should be running on our cluster and scales
-our infrastructure accordingly. The format is described fairly well within the
-yaml file, but an example is below for reference:
+This file defines the resources that should be allocated on the cluster, and it
+is used to scale the infrastructure accordingly. An example is shown below
 
 ```yaml
 training_event:
@@ -18,23 +31,27 @@ training_event:
     end:  2017-10-02
 ```
 
-the label `training_event` is arbitrary but must be unique in the file. We
-specify that we want `count=4` VMs of the flavor c.c10m55 running. If we have
-fewer than this number, this project will launch enough to ensure we're at
-capacity.
+The label `training_event` is arbitrary but must be unique in the file. We
+specify that `count: 4` VMs of the 
+[flavor](https://docs.openstack.org/nova/rocky/user/flavors.html)
+c.c10m55 should be running. If there are fewer than this number, the 
+[vgcn-infrastructure Jenkins project](https://build.galaxyproject.eu/job/usegalaxy-eu/job/vgcn-infrastructure/)
+will launch [`synchronize.py`](synchronize.py) to ensure the actual capacity
+matches the definition.
 
-The `tag` must be specified and is used in constructing the name of the image.
-For a tag of `upload`, images will be named `vgcnbwc-upload-{number}`
+Additionally, `start` and `end` parameters may be supplied. VMs will be 
+launched after the `start` date, and will be killed after the `end` date. This
+permits defining resources ahead of the time they will be needed and releases
+them automatically when they are no longer in use.
 
-We additionally allow supplying a `start` and `end` parameter for the VMs.
-Before this time the VMs will not be launched. After this time, the VMs will
-all be (gracefully) killed on sight. This permits being rather lazy about
-adding and removing defintions and we can be sure that resources are not
-needlessly wasted.
+A formal definition of the schema is available within 
+[`schema.yaml`](schema.yaml).
 
-## Multiple Flavors
+### Multiple flavors
 
-If you want multiple flavors in a group, just create N different groups with different tags. E.g.
+Unfortunately, only a single flavor can be assigned to a group. As a workaround
+to using multiple flavors in a group, just create N different groups with
+different tags. E.g.
 
 ```
 compute_nodes_small:
@@ -47,7 +64,3 @@ compute_nodes_large:
     flavor: c.c24m120
     tag: compute-large
 ```
-
-## LICENSE
-
-GPLv3

add-training.sh

@@ -5,7 +5,7 @@ export LC_NUMERIC="en_US.UTF-8"
 if ([ $# -lt 5 ]); then
     echo "Usage:"
     echo
-    echo "  $0 <training-identifier> <vm-size (e.g. c.c32m240)> <vm-count> <start in YYYY-mm-dd> <end in YYYY-mm-dd> [--donotautocommitpush]"
+    echo "  $0 <training-identifier> <vm-size (e.g. c1.c28m225d50)> <vm-count> <start in YYYY-mm-dd> <end in YYYY-mm-dd> [--donotautocommitpush]"
     echo
     exit 1;
 fi

clouds.yaml

@@ -0,0 +1,57 @@
+$ANSIBLE_VAULT;1.1;AES256
+66666637633663363134313030623361383831366661623638663532323637386563353037633038
+3632326131313961666339373334633163313633303765650a653334333662313265626166623139
+64336634353861373833333537636132393031623062626163353734363532623831343835353635
+3635346237333832390a633430323664666161366166336664623739366330373030366531316133
+37653938636666306134313030363636376533313535353061613137653039633938396536393263
+63363266663236353065383939303962396466303232653636373330616338633134303736613564
+61363362666233633635363161333263646630326432343264363136613734303166623039353837
+30396331356431646665326532353334326530353065326461633639623366363030343533653535
+37356563393965333266333835613930373563643562643838653533636262383437643136636563
+38663263363262633465373434383564373364343763313935336638653332636633363938323661
+65376561663562383137303434303032393438616332303537316433373833636662623763353339
+33336366613734333062636433326364323831366431363836616365353637653738666534316331
+62356263363966653864646137613430633836386166313139653661376564646230316432613261
+61303833613466623062653865393562646637396162373766653161363164346330636232626362
+38336539313230303538663030303866383736653932313663383163386236373936336538616366
+66393039306437626363626563363538353038393836393064363932656236313465636465323362
+34303163396165653366643931653337666136376235623230306434393038646337313730366565
+38306463373861653330613238373962343836383664356338393030623439616564666232346634
+39613736643835393633383262663061623330396566346237383837363639393364373238383263
+61633463626430343434626164376365383135396635633035366562386566626634363037636564
+65363964376639373939383730666434396132346337643038656438313637393032363662636366
+32653430326663346630346564633635366233336164333035326562373131393032326539363238
+62623333346430306538666232356337373962643033373366313662663237336363653963336530
+30653438366437343632373435646366343436366462626563313434626134663835383738326266
+36323233666134653537613462376231386239656464336539336535313738323538323035336165
+39393038303234666332653062333836353132396635306139646466356366346633326439306566
+64663863316437643662343366343431363638646164373563623663336234613139623931653738
+63383631633839373439343862373830653934616365343337343563316365346337663739343934
+32633866636161363234323535313739633366626263333933636635616239643532336266313561
+34643566623632653431323630386538386430373161623731333939616364333164323566396164
+65323130386230643062663531383330663364663630363766653932643864626339623832373432
+32666161626162643164633333376139316531326163643534636462383961623763613661386635
+36656262633738646234333938656530646332336636326366316266653138386236346434323533
+39633963666361313161306139663137326538326232366237336363353232383332306163623066
+37396332633530643739383639303838653534376264626462366434643166313838663133616439
+35666264616138383536333965356464663935643237363330363437653135626637343932636463
+66383234643937663562393338623036643832666237316363626438323637396262376264333938
+63376137323162373564323763343932363939393263663335623832636363663335313064376466
+63656263613538613563373739333530663361636432646430393338393030343936663964363264
+31333365616136343836366465663433316532333064623534616565393431353065383632663938
+39333534653238653964386361383236623066333262623331386532346638653734643639363832
+38316464623835663830396662643530626333303436396261623037343264643666646236383761
+33323764653834303339363931306136616235366539313737336239616431343830366661396530
+64636365373234363035376531313063373138326637336265313539303735396339633531376564
+31393333336434333163346431366131623136353938313032346163323630326436633533396134
+65373632343538366331656638316630393934333633613432643933663631613061313661306363
+31616538383935653166666338653336396432396563333737313734336235366534303565663564
+37396336386538336331646534373331333533313232613461313032623065346338376338613162
+34356333626638316631306261333363353262313134316630393532306231626664303730376365
+63383332613364616436646366356563323130303663316464623465626264306233376537373535
+66353265626134346132643234666363303135306562356233643839316539623634373333393631
+62383735636133373861663935343535363163613962656666383730376638363737343431393261
+39383931323163326132363131383430613439386163646333663033396566303233366265313133
+30623066303138613762616133393764306163613265346333323334306565613936626632323761
+34646466633030383839393234653236346661353437613765373834636532643837323463626636
+31306565373163346438