Merge pull request #47 from thin-edge/test-isolated-tests

test: use isolated container engines

.dockerignore

@@ -0,0 +1 @@
+tests/*.tar.gz

.github/workflows/test.yaml

@@ -23,26 +23,38 @@ jobs:
         run: echo "Approved"
 
   test:
-    name: Test ${{ matrix.job.target }}
+    name: Test ${{ matrix.job.name }}
     runs-on: ubuntu-latest
     needs: approve
     environment:
       name: Test Auto
     env:
-      COMPOSE_PROJECT_NAME: ci_${{ matrix.job.target }}_${{github.run_id}}_${{github.run_attempt || '1'}}
-      DEVICE_ID: ci_${{ matrix.job.target }}_${{github.run_id}}_${{github.run_attempt || '1'}}
+      TEST_IMAGE: ${{ matrix.job.image }}
 
     strategy:
       fail-fast: false
       matrix:
         job:
-          - { target: tedge-container-bundle }
+          # docker
+          - { target: tedge-container-bundle, name: "docker v27", image: "docker:27-dind" }
+          - { target: tedge-container-bundle, name: "docker v26", image: "docker:26-dind" }
+          - { target: tedge-container-bundle, name: "docker v25", image: "docker:25-dind" }
+          - { target: tedge-container-bundle, name: "docker v20", image: "docker:20-dind" }
+          # podman
+          - { target: tedge-container-bundle, name: "podman", image: debian-systemd-podman-cli }
+          - { target: tedge-container-bundle, name: "podman v5", image: "podman-v5" }
+          # - { target: tedge-container-bundle, name: "podman v4", image: "podman-v4" }
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+
       - uses: taiki-e/install-action@just
 
       - uses: reubenmiller/setup-go-c8y-cli@main
@@ -53,20 +65,11 @@ jobs:
         run: |
           touch .env
 
-          echo "DEVICE_ID=$DEVICE_ID" >> .env
           echo 'C8Y_BASEURL="${{ secrets.C8Y_BASEURL }}"' >> .env
-          C8Y_DOMAIN=$(echo "${{ secrets.C8Y_BASEURL }}" | sed 's|.*://||g')
           echo 'C8Y_USER="${{ secrets.C8Y_USER }}"' >> .env
           echo 'C8Y_PASSWORD="${{ secrets.C8Y_PASSWORD }}"' >> .env
-
-          # env variables required by the container
-          echo "TEDGE_C8Y_URL=$C8Y_DOMAIN" >> .env
           cat .env
 
-      - name: Bootstrap device
-        run: |
-          just init
-
       - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
@@ -77,10 +80,7 @@ jobs:
       - name: Install dependencies
         run: |
           just venv
-
-      - name: Start demo
-        run: |
-          just start -d
+          just build-test
 
       - name: Run tests
         run: just test
@@ -89,18 +89,9 @@ jobs:
         uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: reports
+          name: reports-${{ matrix.job.name }}
           path: output
 
-      - name: Stop demo
-        if: always()
-        run: just stop -v
-
-      - name: Cleanup Devices
-        if: always()
-        run: |
-          just cleanup "$DEVICE_ID"
-
       - name: Send report to commit
         uses: joonvena/[email protected]
         with:

.gitignore

@@ -1,7 +1,15 @@
 .env
-device-certs/
+
+# Artifacts
 *.tar
+
+# python
 .venv/
+
+# Test
 output/
 *.html
 *.xml
+
+# Test artifacts
+tests/*.tar.gz

README.md

@@ -42,35 +42,21 @@ TEDGE_C8Y_OPERATIONS_AUTO_LOG_UPLOAD=always
 
 ### Development
 
-#### Starting the container
+#### Testing
 
-The following tools are required to run the container:
+The system tests are writing using the [RobotFramework](https://robotframework.org/) with some custom thin-edge.io libraries. Generally the test framework will spin up a new container engine environment (defined by the `TEST_IMAGE` variable). In the test itself, a new instance of the **tedge-container-bundle** will be created which the test then uses to check the specified functionality. Using this setup does bring a but if complexity into the setup, however it is necessary to ensure that the **tedge-container-bundle** can be tested against multiple container engine environments (e.g. docker, podman and different versions of each), whilst it also provides a test environment which does not pollute your host's container engine environment.
+
+The following tools are required to run the tests:
 
 * docker
-* docker compose
+* [docker-buildx-plugin](https://github.com/docker/buildx)
+* [go-c8y-cli](https://goc8ycli.netlify.app/)
 * Optional: [just](https://github.com/casey/just) - used to run project tasks
+* python >= 3.9
 
 After the project pre-requisites have been installed, you can start the container using the following steps:
 
-1. Create a `.env` file containing the environment variables (see the [Providing the device certificate by environment variables](./README.md#providing-the-device-certificate-by-environment-variables) for details on how to provide the device certificate)
-
-    ```sh
-    # device id to use for the certificate
-    DEVICE_ID=demo01
-
-    # Which c8y instance you want to connect to
-    TEDGE_C8Y_URL=example.cumulocity.com
-
-    # You can turn specific services on/off via environment variables
-    SERVICE_TEDGE_MAPPER_AWS=0
-    SERVICE_TEDGE_MAPPER_AZ=0
-    SERVICE_TEDGE_MAPPER_C8Y=1
-
-    # Other settings
-    TEDGE_C8Y_OPERATIONS_AUTO_LOG_UPLOAD=always
-    ```
-
-2. Activate your Cumulocity session using go-c8y-cli
+1. First run only: Activate your Cumulocity session using [go-c8y-cli](https://goc8ycli.netlify.app/docs/gettingstarted/#creating-a-new-session)
 
     ```sh
     set-session
@@ -87,22 +73,32 @@ After the project pre-requisites have been installed, you can start the containe
     set-session
     ```
 
-3. Init the device certificate (stored in a container volume) and upload it to Cumulocity IoT
+2. Create a `.env` file containing the environment variables used for testing
 
     ```sh
-    just init
+    just init-dotenv
     ```
 
-4. Start the container (using docker compose)
+    **Note** This will write your current go-c8y-cli's session credentials to the `.env` file, so you don't need to use `set-session` every time.
+
+3. Initialize the docker setup and install the python virtual environment
 
     ```sh
-    # using the justfile task
-    just start
+    just build-setup
+    just venv
+    ```
 
-    # Or using docker compose directly
-    docker compose up --build
+4. Build test images
+
+    ```sh
+    just build-test
     ```
 
+5. Run system tests
+
+    ```sh
+    just test
+    ```
 
 ## Project structure
 
@@ -118,3 +114,23 @@ After the project pre-requisites have been installed, you can start the containe
 * Both the initialization scripts and services have access to the container's environment variables, which makes it much easier to configure the components.
 * Standard Output and Standard Error are redirected to the PID 1 so that the log messages are visible from all services
 * Run services under the container `USER`
+
+## Contributions
+
+#### PR Submissions
+
+Contributions are welcomed, but please consider the following
+
+* Does the change make sense for other users? If the answer is no, then maybe you should be just pulling in the `tedge-container-bundle` into your own Dockerfile using `FROM ghcr.io/thin-edge/tedge-container-bundle:<tag>`
+
+* State the motivation of the change
+
+* Write a system test (under `./tests`) and ensure all tests are passing (though the CI will also run this on the PR directly)
+
+Finally, before submitting a PR you should run the following locally to ensure everything is formatted correct and there are no linting errors/warnings!
+
+```sh
+just venv
+just lint
+just format
+```

files/tedge/container-logs.sh

@@ -68,13 +68,13 @@ TMP_FILE="${TMP_LOG_DIR}/${TYPE}_${CONTAINER_NAME}_$(date -Iseconds).log"
     echo "dateFrom:   $DATE_FROM"
     echo "dateTo:     $DATE_TO"
     echo "maxLines:   $MAX_LINES"
-    echo "command:    $DOCKER_CMD logs -n \"$MAX_LINES\" --since \"$DATE_FROM\" --until \"$DATE_TO\" \"$CONTAINER_NAME\""
+    echo "command:    $DOCKER_CMD logs --tail \"$MAX_LINES\" --since \"$DATE_FROM\" --until \"$DATE_TO\" \"$CONTAINER_NAME\""
     echo "------------------------------------------------------"
     echo
 } > "$TMP_FILE"
 
 # Write logs to file (stripping any ansci colour codes)
-$DOCKER_CMD logs -n "$MAX_LINES" --since "$DATE_FROM" --until "$DATE_TO" "$CONTAINER_NAME" 2>&1 \
+$DOCKER_CMD logs --tail "$MAX_LINES" --since "$DATE_FROM" --until "$DATE_TO" "$CONTAINER_NAME" 2>&1 \
     | sed -e 's/\x1b\[[0-9;]*m//g' \
     | tee -a "$TMP_FILE"
 

files/tedge/container_run.tpl

@@ -2,7 +2,6 @@
     golang template to build the docker run options using `docker inspect <container> --format "container_run.tpl"`
     Credit: https://gist.github.com/efrecon/8ce9c75d518b6eb863f667442d7bc679
 */}}
-  --name {{printf "%q" .Name}} \
     {{- with .HostConfig}}
         {{- if .Privileged}}
   --privileged \
@@ -45,9 +44,6 @@
         {{- if .PublishAllPorts}}
   --publish-all \
         {{- end}}
-        {{- if .UTSMode}}
-  --uts {{printf "%q" .UTSMode}} \
-        {{- end}}
   --restart always \
         {{- range $e := .ExtraHosts}}
   --add-host {{printf "%q" $e}} \