[DOC-603] Simplify Toolchain Configuration handling

.circleci/base_config.yml

@@ -426,7 +426,7 @@ jobs:
             - "28:4f:0b:b7:71:cd:39:ab:c0:33:65:4f:da:6a:cc:8e"
             - "bc:4d:f3:df:53:ad:22:d8:60:5d:10:d8:fd:74:d2:cd"
       - attach_workspace:
-          at: .
+          at: /tmp
 
       - run: ssh-keyscan github.com >> ~/.ssh/known_hosts && cat ~/.ssh/known_hosts
       - clone-docs-and-merge

.circleci/generate_config.py

@@ -280,7 +280,9 @@ def workflow_generate_launch_command(config):
 export HUGO_URL=https://<< pipeline.parameters.deploy-url >>--docs-hugo.netlify.app\n \
 export HUGO_ENV=examples\n \
 export OVERRIDE=<< pipeline.parameters.override >>\n \
-export GENERATORS='<< parameters.generators >>'\n"
+: > /home/circleci/project/docs-hugo/toolchain/docker/config.yaml\n   \
+echo 'generators: << parameters.generators >>' >> /home/circleci/project/docs-hugo/toolchain/docker/config.yaml\n\
+echo 'servers:' >> /home/circleci/project/docs-hugo/toolchain/docker/config.yaml"
 
     for i in range(len(versions)):
         version = versions[i]["name"]
@@ -296,8 +298,9 @@ def workflow_generate_launch_command(config):
 
         version_underscore = version.replace(".", "_")
         branchEnv = f"{pullImage}\n \
-export ARANGODB_BRANCH_{version_underscore}={branch}\n \
-export ARANGODB_SRC_{version_underscore}=/home/circleci/project/{version}"
+echo '  \"{version}\": \"{branch}\"' >>  /home/circleci/project/docs-hugo/toolchain/docker/config.yaml\n\
+mkdir -p /tmp/arangodb\n\
+mv /tmp/{version} /tmp/arangodb/"
 
         shell = f"{shell}\n{branchEnv}"
 
@@ -371,14 +374,18 @@ def workflow_release_launch_command(config):
 export ENV=\"circleci\"\n \
 export HUGO_URL=https://docs.arangodb.com\n \
 export HUGO_ENV=release\n \
-export GENERATORS=''\n"
+: > /home/circleci/project/docs-hugo/toolchain/docker/config.yaml\n   \
+echo 'generators: \"\"' >> /home/circleci/project/docs-hugo/toolchain/docker/config.yaml\n\
+echo 'servers:' >> /home/circleci/project/docs-hugo/toolchain/docker/config.yaml"
 
     pullImage = pullImageCmd(args.arangodb_branch, args.docs_version)
 
     version_underscore = args.docs_version.replace(".", "_")
     branchEnv = f"{pullImage}\n \
-export ARANGODB_BRANCH_{version_underscore}={args.arangodb_branch}\n \
-export ARANGODB_SRC_{version_underscore}=/home/circleci/project/{args.docs_version}"
+echo '  \"{args.docs_version}\": \"{args.arangodb_branch}\"' >>  /home/circleci/project/docs-hugo/toolchain/docker/config.yaml\n\
+mkdir -p /tmp/arangodb\n\
+mv /tmp/{args.docs_version} /tmp/arangodb/"
+
 
     shell = f"{shell}\n{branchEnv}"
 
@@ -402,7 +409,7 @@ def pullImageCmd(branch, version):
 version={version}\n"
         pullImage += "\
 image_name=$(echo ${BRANCH##*/})\n\
-main_hash=$(awk 'END{print}' $version/.git/logs/HEAD | awk '{print $2}' | cut -c1-9)\n\
+main_hash=$(awk 'END{print}' /tmp/$version/.git/logs/HEAD | awk '{print $2}' | cut -c1-9)\n\
 docker pull arangodb/docs-hugo:$image_name-$version-$main_hash\n\
 docker tag arangodb/docs-hugo:$image_name-$version-$main_hash $image_name-$version"
 

README.md

@@ -183,10 +183,9 @@ The toolchain container needs to be set up via config file in `toolchain/docker/
 
 ```yaml
 generators:   # Generators to trigger - empty string defaults to all generators
-servers:      # Array to define arangodb servers to be used by the toolchain
-  - image:    # arangodb docker image to be used, can be arangodb/enterprise-preview:... or a branch name
-    version:  # docs branch to put the generated content into
-  - ...       # Additional images and versions as needed
+servers:      # define arangodb servers to be used by the toolchain
+  "{docs-version}": "{arangodb-branch}"    # version folder of the documentation to put the generated content into: arangodb docker image to be used, can be arangodb/enterprise-preview:... or a branch name
+  ...
 ```
 
 **Available generators**
@@ -204,26 +203,22 @@ If `metrics` or `error-codes` is in the `generators` string, the following
 environment variable has to be exported:
 
 ```shell
-export ARANGODB_SRC_{VERSION}=path/to/arangodb/source
+export ARANGODB_SRC=path/to/arangodb/source
 ```
 
-Substitute `{VERSION}` with a version number like `3_11`.
-
 On Windows using PowerShell, use a Unix-like path:
 
 ```powershell
-$Env:ARANGODB_SRC_3_11 = "/Drive/path/to/arangodb"
+$Env:ARANGODB_SRC = "/Drive/path/to/arangodb"
 ```
 
 **Configuration example**
 
 ```yaml
 generators: examples oasisctl options optimizer
 servers:
-  - image: arangodb/enterprise-preview:3.11-nightly
-    version: "3.11"
-  - image: arangodb/enterprise-preview:devel-nightly
-    version: "3.12"
+  "3.11": "arangodb/enterprise-preview:3.11-nightly"
+  "3.12": "arangodb/enterprise-preview:devel-nightly"
 ```
 
 **Run the toolchain**
@@ -895,60 +890,7 @@ It makes a warning show at the top of every page for that version.
    +              --arangodb-branches << pipeline.parameters.arangodb-3_11 >> << pipeline.parameters.arangodb-3_12 >> << pipeline.parameters.arangodb-4_0 >> \
    ```
 
-3. In the `toolchain/docker/amd64/docker-compose.yml` file, add an entry under
-   `services.toolchain.volumes` for the new version. Simply increment the value
-   after `:-/tmp/`. Example:
-
-   ```diff
-          - ${ARANGODB_SRC_3_12:-/tmp/2}:/tmp/3.12
-   +      - ${ARANGODB_SRC_4_0:-/tmp/3}:/tmp/4.0
-   ```
-
-   Under `services.toolchain.environment`, you need to add two different entries
-   for the new version. Example:
-
-   ```diff
-          ARANGODB_SRC_3_11: ${ARANGODB_SRC_3_11}
-          ARANGODB_SRC_3_12: ${ARANGODB_SRC_3_12}
-   +      ARANGODB_SRC_4_0: ${ARANGODB_SRC_4_0}
-          ARANGODB_BRANCH_3_11: ${ARANGODB_BRANCH_3_11}
-          ARANGODB_BRANCH_3_12: ${ARANGODB_BRANCH_3_12}
-   +      ARANGODB_BRANCH_4_0: ${ARANGODB_BRANCH_4_0}
-   ```
-
-   The same changes are required in the
-   `toolchain/docker/arm64/docker-compose.yml` file.
-
-4. In the `toolchain/docker/config.yaml` file, add an entry for the new version.
-   Example:
-
-   ```diff
-      - image: ${ARANGODB_BRANCH_3_12_IMAGE}
-        version: ${ARANGODB_BRANCH_3_12_VERSION}
-   +
-   +  - image: ${ARANGODB_BRANCH_4_0_IMAGE}
-   +    version: ${ARANGODB_BRANCH_4_0_VERSION}
-   ```
-
-5. In the `toolchain/scripts/toolchain.sh` file, find the code that accesses
-   environment variables with the format `$ARANGODB_BRANCH_X_XX` where `X_XX`
-   is a version number like `3_12`, so `$ARANGODB_BRANCH_3_12` for instance.
-   Duplicate the block of an existing version and adjust all version numbers.
-   Example:
-
-   ```diff
-    if [ "$ARANGODB_BRANCH_3_12" != "" ] ; then
-          export ARANGODB_BRANCH_3_12_IMAGE="$ARANGODB_BRANCH_3_12"
-          export ARANGODB_BRANCH_3_12_VERSION="3.12"
-    fi
-
-   +if [ "$ARANGODB_BRANCH_4_0" != "" ] ; then
-   +      export ARANGODB_BRANCH_4_0_IMAGE="$ARANGODB_BRANCH_4_0"
-   +      export ARANGODB_BRANCH_4_0_VERSION="4.0"
-   +fi
-   ```
-
-6. In the `site/data` folder, create a new folder with the short version number
+3. In the `site/data` folder, create a new folder with the short version number
    as the name, e.g. `4.0`. In the new `site/data/4.0` folder, create a
    `cache.json` file with the following content:
 
@@ -958,7 +900,7 @@ It makes a warning show at the top of every page for that version.
 
    Add this untracked file to Git!
 
-7. Duplicate the folder of the most recent version in `site/content`, e.g.
+4. Duplicate the folder of the most recent version in `site/content`, e.g.
    the `3.12` folder, and rename the copy to the new version, e.g. `4.0`.
 
    The `menuTitle` in the front matter of the version homepage, e.g.
@@ -1001,7 +943,7 @@ It makes a warning show at the top of every page for that version.
 
    Add the new, untracked files to Git!
 
-8. In the `PULL_REQUEST_TEMPLATE.md` file, add a new line for the new version.
+5. In the `PULL_REQUEST_TEMPLATE.md` file, add a new line for the new version.
    Example:
 
    ```diff
@@ -1017,7 +959,7 @@ It makes a warning show at the top of every page for that version.
 
    Expect the plain build to fail for the time being because of missing data files.
 
-9. You can use CircleCI to initially generate the data files for the new version,
+6. You can use CircleCI to initially generate the data files for the new version,
    like the startup option dumps. You can also populate the example cache at the
    same time.
 

toolchain/arangoproxy/go.mod

@@ -5,5 +5,4 @@ go 1.18
 require (
 	github.com/dlclark/regexp2 v1.7.0
 	gopkg.in/yaml.v3 v3.0.1
-)
-
+)

toolchain/docker/amd64/docker-compose.yml

@@ -5,20 +5,13 @@ services:
     image: arangodb/docs-hugo:toolchain-amd64
     tty: true
     volumes:
-      - ${ARANGODB_SRC_3_10:-/dev/null}:/tmp/3.10
-      - ${ARANGODB_SRC_3_11:-/tmp/1}:/tmp/3.11
-      - ${ARANGODB_SRC_3_12:-/tmp/2}:/tmp/3.12
+      - ${ARANGODB_SRC:-/dev/null}:/tmp/arangodb_src
+      - /tmp/arangodb:/tmp/arangodb
       - ../../../:/home/
       - /var/run/docker.sock:/var/run/docker.sock
     environment:
       ENV: ${ENV:-local}
       HUGO_URL: ${HUGO_URL:-http://localhost}
       HUGO_ENV: ${HUGO_ENV:-development}
-      GENERATORS: ${GENERATORS}
       OVERRIDE: ${OVERRIDE}
-      ARANGODB_SRC_3_10: ${ARANGODB_SRC_3_10}
-      ARANGODB_SRC_3_11: ${ARANGODB_SRC_3_11}
-      ARANGODB_SRC_3_12: ${ARANGODB_SRC_3_12}
-      ARANGODB_BRANCH_3_10: ${ARANGODB_BRANCH_3_10}
-      ARANGODB_BRANCH_3_11: ${ARANGODB_BRANCH_3_11}
-      ARANGODB_BRANCH_3_12: ${ARANGODB_BRANCH_3_12}
+      ARANGODB_SRC: ${ARANGODB_SRC}

toolchain/docker/arm64/docker-compose.yml

@@ -5,20 +5,13 @@ services:
     image: arangodb/docs-hugo:toolchain-arm64
     tty: true
     volumes:
-      - ${ARANGODB_SRC_3_10:-/dev/null}:/tmp/3.10
-      - ${ARANGODB_SRC_3_11:-/tmp/1}:/tmp/3.11
-      - ${ARANGODB_SRC_3_12:-/tmp/2}:/tmp/3.12
+      - ${ARANGODB_SRC:-/dev/null}:/tmp/arangodb_src
+      - /tmp/arangodb:/tmp/arangodb
       - ../../../:/home/
       - /var/run/docker.sock:/var/run/docker.sock
     environment:
       ENV: ${ENV:-local}
       HUGO_URL: ${HUGO_URL:-http://localhost}
       HUGO_ENV: ${HUGO_ENV:-development}
-      GENERATORS: ${GENERATORS}
       OVERRIDE: ${OVERRIDE}
-      ARANGODB_SRC_3_10: ${ARANGODB_SRC_3_10}
-      ARANGODB_SRC_3_11: ${ARANGODB_SRC_3_11}
-      ARANGODB_SRC_3_12: ${ARANGODB_SRC_3_12}
-      ARANGODB_BRANCH_3_10: ${ARANGODB_BRANCH_3_10}
-      ARANGODB_BRANCH_3_11: ${ARANGODB_BRANCH_3_11}
-      ARANGODB_BRANCH_3_12: ${ARANGODB_BRANCH_3_12}
+      ARANGODB_SRC: ${ARANGODB_SRC}

toolchain/docker/config.yaml

@@ -1,10 +1,2 @@
-generators: ${GENERATORS}
-servers:
-  - image: ${ARANGODB_BRANCH_3_10_IMAGE}
-    version: ${ARANGODB_BRANCH_3_10_VERSION}
-
-  - image: ${ARANGODB_BRANCH_3_11_IMAGE}
-    version: ${ARANGODB_BRANCH_3_11_VERSION}
-
-  - image: ${ARANGODB_BRANCH_3_12_IMAGE}
-    version: ${ARANGODB_BRANCH_3_12_VERSION}
+generators: ""
+servers: ""