Merge pull request #1 from thin-edge/chore-update-thin-edge.io

chore: update thin-edge.io and add tests

.github/workflows/publish.yaml

@@ -0,0 +1,37 @@
+name: Publish
+
+on:
+  workflow_dispatch:
+    inputs:
+      VERSION:
+        description: "Version"
+        type: string
+        default: ""
+  push:
+    tags:
+      - "*"
+
+jobs:
+  push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: taiki-e/install-action@just
+
+      - name: Publish images
+        run: just build registry ${{ inputs.VERSION || github.ref_name }}

.github/workflows/test.yaml

@@ -0,0 +1,111 @@
+name: Test
+
+on:
+  workflow_dispatch:
+  # Use a manual approval process before PR's are given access to
+  # the secrets which are required to run the integration tests.
+  # The PR code should be manually approved to see if it can be trusted.
+  # When in doubt, do not approve the test run.
+  # Reference: https://dev.to/petrsvihlik/using-environment-protection-rules-to-secure-secrets-when-building-external-forks-with-pullrequesttarget-hci
+  pull_request_target:
+    branches: [ main ]
+  merge_group:
+jobs:
+  approve:
+    name: Approve
+    environment:
+      # For security reasons, all pull requests need to be approved first before granting access to secrets
+      # So the environment should be set to have a reviewer/s inspect it before approving it
+      name: ${{ github.event_name == 'pull_request_target' && 'Test Pull Request' || 'Test Auto'  }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Wait for approval
+        run: echo "Approved"
+
+  test:
+    name: Test ${{ matrix.job.target }}
+    runs-on: ubuntu-latest
+    needs: approve
+    environment:
+      name: Test Auto
+    env:
+      COMPOSE_PROJECT_NAME: ci_${{ matrix.job.target }}_${{github.run_id}}_${{github.run_attempt || '1'}}
+      DEVICE_ID: ci_${{ matrix.job.target }}_${{github.run_id}}_${{github.run_attempt || '1'}}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        job:
+          - { target: tedge-container-bundle }
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}
+
+      - uses: taiki-e/install-action@just
+
+      - uses: reubenmiller/setup-go-c8y-cli@main
+      - name: install c8y-tedge extension
+        run: c8y extension install thin-edge/c8y-tedge
+
+      - name: create .env file
+        run: |
+          touch .env
+          just init "$DEVICE_ID"
+
+          echo "DEVICE_ID=$DEVICE_ID" >> .env
+          echo 'C8Y_BASEURL="${{ secrets.C8Y_BASEURL }}"' >> .env
+          C8Y_DOMAIN=$(echo "${{ secrets.C8Y_BASEURL }}" | sed 's|.*://||g')
+          echo 'C8Y_USER="${{ secrets.C8Y_USER }}"' >> .env
+          echo 'C8Y_PASSWORD="${{ secrets.C8Y_PASSWORD }}"' >> .env
+
+          # env variables required by the container
+          echo "TEDGE_C8Y_URL=$C8Y_DOMAIN" >> .env
+          cat .env
+
+      - name: Upload certificate
+        run: |
+          just upload
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+          cache-dependency-path: |
+            tests/requirements.txt
+
+      - name: Install dependencies
+        run: |
+          just venv
+
+      - name: Start demo
+        run: |
+          export DOCKER_USER="1000:101"
+          just start -d
+
+      - name: Run tests
+        run: just test
+
+      - name: Upload test results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: reports
+          path: output
+
+      - name: Stop demo
+        if: always()
+        run: just stop -v
+
+      - name: Cleanup Devices
+        if: always()
+        run: |
+          just cleanup "$DEVICE_ID"
+
+      - name: Send report to commit
+        uses: joonvena/[email protected]
+        with:
+          gh_access_token: ${{ secrets.GITHUB_TOKEN }}
+          report_path: 'output'
+          show_passed_tests: 'true'

.gitignore

@@ -1 +1,5 @@
 .env
+device-certs/
+*.tar
+.venv/
+output/

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python3",
+    "robot.language-server.python": "${workspaceFolder}/.venv/bin/python3",
+    "robot.python.executable": "${workspaceFolder}/.venv/bin/python3",
+    "python.envFile": "${workspaceFolder}/.env"
+}

Dockerfile

@@ -1,7 +1,9 @@
-ARG VERSION=1.1.1
-FROM ghcr.io/thin-edge/tedge:${VERSION}
-ARG TARGETARCH
-ARG S6_OVERLAY_VERSION=3.1.6.2
+ARG TEDGE_TAG=1.2.0
+# thin-edge.io base image name: tedge, tedge-main
+ARG TEDGE_IMAGE=tedge
+FROM ghcr.io/thin-edge/${TEDGE_IMAGE}:${TEDGE_TAG}
+ARG TARGETPLATFORM
+ARG S6_OVERLAY_VERSION=3.2.0.0
 
 USER root
 
@@ -18,11 +20,12 @@ RUN apk update \
 
 # Install s6-overlay
 # Based on https://github.com/just-containers/s6-overlay#which-architecture-to-use-depending-on-your-targetarch
-RUN case ${TARGETARCH} in \
-        "amd64")  S6_ARCH=x86_64  ;; \
-        "arm64")  S6_ARCH=aarch64  ;; \
-        "arm/v6")  S6_ARCH=armhf  ;; \
-        "arm/v7")  S6_ARCH=arm  ;; \
+RUN case ${TARGETPLATFORM} in \
+        "linux/amd64")  S6_ARCH=x86_64  ;; \
+        "linux/arm64")  S6_ARCH=aarch64  ;; \
+        "linux/arm/v6")  S6_ARCH=armhf  ;; \
+        "linux/arm/v7")  S6_ARCH=arm  ;; \
+        *) echo "Unsupported target platform: TARGETPLATFORM=$TARGETPLATFORM"; exit 1 ;; \
     esac \
     && curl https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-noarch.tar.xz -L -s --output /tmp/s6-overlay-noarch.tar.xz \
     && tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz \
@@ -53,18 +56,23 @@ COPY files/tedge/tedge.toml /etc/tedge/
 COPY files/tedge/plugins/*.toml /etc/tedge/plugins/
 
 
-ENV S6_BEHAVIOUR_IF_STAGE2_FAILS=2
-ENV S6_CMD_WAIT_FOR_SERVICES_MAXTIME=30000
+ENV S6_BEHAVIOUR_IF_STAGE2_FAILS 2
+ENV S6_CMD_WAIT_FOR_SERVICES_MAXTIME 30000
 
 
 # Control which mappers are running
 # You can see the list of thin-edge.io services and the related env variable
 # in the service definition under:
 # * https://github.com/thin-edge/tedge-services/tree/main/services/s6-overlay/s6-rc.d
-ENV SERVICE_TEDGE_MAPPER_AWS=0
-ENV SERVICE_TEDGE_MAPPER_AZ=0
-ENV SERVICE_TEDGE_MAPPER_C8Y=1
-ENV SERVICE_TEDGE_MAPPER_COLLECTD=0
+ENV SERVICE_MOSQUITTO 1
+
+ENV SERVICE_TEDGE_MAPPER_AWS 0
+ENV SERVICE_TEDGE_MAPPER_AZ 0
+ENV SERVICE_TEDGE_MAPPER_C8Y 1
+ENV SERVICE_TEDGE_MAPPER_COLLECTD 0
+
+ENV SERVICE_TEDGE_AGENT 1
+ENV SERVICE_C8Y_FIRMWARE_PLUGIN 0
 
 
 # Control thin-edge.io settings via env variables

README.md

@@ -32,10 +32,6 @@ After the project pre-requisites have been installed, you can start the containe
 1. Create a `.env` file containing the environment variables (see the [Providing the device certificate by environment variables](./README.md#providing-the-device-certificate-by-environment-variables) for details on how to provide the device certificate)
 
     ```sh
-    # Device certificate (public/private)
-    CERTPRIVATE=<base64_encoded_private_key>
-    CERTPUBLIC=<base64_encoded_public_cert>
-
     # Which c8y instance you want to connect to
     TEDGE_C8Y_URL=example.cumulocity.com
 
@@ -45,7 +41,19 @@ After the project pre-requisites have been installed, you can start the containe
     SERVICE_TEDGE_MAPPER_C8Y=1
     ```
 
-2. Start the container (using docker compose)
+2. Init the device certificate (stored under `./device-cert)
+
+    ```sh
+    just init "$DEVICE_ID"
+    ```
+
+3. Upload the device certificate to Cumulocity IoT
+
+    ```sh
+    just upload
+    ```
+
+4. Start the container (using docker compose)
 
     ```sh
     # using justfile task
@@ -61,29 +69,6 @@ After the project pre-requisites have been installed, you can start the containe
     just run-container
     ```
 
-### Providing the device certificate by environment variables
-
-The thin-edge.io certificate (both public and private keys) are provided to the container via environment variables.
-
-The following environment variables are used to provide the certificate to the container's startup script `50_configure.sh` which is called by s6-overlay when the container starts up:
-
-* `CERTPRIVATE` - Device certificate private key (base64 encoded)
-* `CERTPUBLIC` - Device certificate public key (base64 encoded)
-
-The environment variables are base64 encoded to avoid any shell quoting problems with whitespace any other unexpected characters. If you have the certificate files (public and private key), then you generate the encoded values using the following one-liners:
-
-```sh
-cat /etc/tedge/device-certs/tedge-private-key.pem | base64 | tr -d '\n' | xargs printf 'CERTPRIVATE=%s\n'
-cat /etc/tedge/device-certs/tedge-certificate.pem | base64 | tr -d '\n' | xargs printf 'CERTPUBLIC=%s\n'
-```
-
-You can copy the output to your `.env` file (or set the environment variables yourself). An example of the `.env` file is shown below:
-
-```sh
-CERTPRIVATE=LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tsS0tCk1JR0hBZ0bBTUJNR0J5cUdTTTQ5QWdFR0NDcUdTTTQ5QXdFSEJHMHdhd0lCQVFRZ2lHcDE3eEZ5VlcvZXlka1kKaE4rM05McWtMM3dIK0d3c1BSZnFmZk1NQU9taFJBTkNBQVEyRVhSNnFnb3JNcldPUzQyNVlRT21DbFVsWWZHdwp6alRySnF6WnZjOTVkTzJnNUZEb1Z4ZFZSUuc5MWJNOFNvdDFVWnlZMklEaXpSUzBHZ3c0NgCBQUklWQVRFIEtFWS0tLS0tCd==
-CERTPUBLIC=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJ4RENDQVdxZ0F3SUJBZ0lVRlFOelh5eUNqTEpMQmxQVWRJeWUzNW9pUzZNd0NnWUlLb1pJemowRUSXcKU0RFY01Cb0dBMVVFQXd3VGRHVmtaMlZmWTI5dWRHRnBibVZ5WHpBd01URVNNQkFHQTFVRUNnd0pWR2hwYmlgpaR2RsTVJRd0VnWURVFMREF0VVpYTjBJRVJsZG1salpUQWVGdzB5TkRBMk1EWXdPREV3TXpaYUZ3MHlOVEEyCk1EWXdPREV3TXpaYU1FZ3hIREFhQmdkJBTU1FM1JsWkdkbFgyTnZiblJoYVc1bGNsOHdNREV4RWpBUUJnTlYKQkFvTUNWUm9hVzRnUldSblpURVVNQklHQTFVRUN3d0xWR1Z6NCRVpYWnBZMlV3V1RBVEJnY3Foa2pPUFFJQgpCZ2dxaGtqT1BRTUJCd05DQUFRMkVjZxZ29yTXJXT1M0MjVZUU9tQ2xVbFlmR3d6alRySnF6WnZjOTVkeTluCk8yZzVGRG9WeGRWUlFHOTFiTThTL3QxVVp5WTJJRGl6UlMwR2d3NDZvekl3TURBZEJnTlZIUTRFRmdRVUZRTnoKWHl5Q2pMSkxCbFBVZEl5ZTM1b2lTNk13ROSQpBREJGQWlFQSs1Z09Gem5LQ0lNSFVWODFJVzhMZkhFUzE0SXdGR3Y3Q2dzZjd2ZzNhck1DSUR1cGpsRjBFaHozCkx6d1VURySTR6YmRGbzc1WWVKODBXNXg1V1hlZzEzCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
-```
-
 ## Project structure
 
 |Directory|Description|

cont-init.d/50_configure.sh

@@ -1,35 +1,35 @@
 #!/command/with-contenv sh
+# shellcheck shell=sh
 set -e
-#
-# Convert environment variables config to files
-#
 echo "Current User: $(whoami)"
 
-if [ -n "$CERTPRIVATE" ]; then
-    echo "Writing thin-edge.io private key from env 'CERTPRIVATE' (decoding from base64) to file" >&2
-    CERT_FILE_KEY="$(tedge config get device.key_path)"
-    printf '%s' "$CERTPRIVATE" | tr -d '"' | base64 -d > "$CERT_FILE_KEY"
+#
+# Note: Due to permissions problems, copy the secrets from the /run read-only path to /etc/tedge/device-certs/
+#
+CERT_FILE_KEY="$(tedge config get device.key_path)"
+if [ -f /run/secrets/certificate_private_key ]; then
+    cat /run/secrets/certificate_private_key > "$CERT_FILE_KEY"
     chmod 600 "$CERT_FILE_KEY"
 fi
 
-
-if [ -n "$CERTPUBLIC" ]; then
-    echo "Writing thin-edge.io private key from env 'CERTPUBLIC' (decoding from base64) to file" >&2
-    CERT_FILE_PUB="$(tedge config get device.cert_path)"
-    printf '%s' "$CERTPUBLIC" | tr -d '"' | base64 -d > "$CERT_FILE_PUB"
+CERT_FILE_PUB="$(tedge config get device.cert_path)"
+if [ -f /run/secrets/certificate_public_key ]; then
+    cat /run/secrets/certificate_public_key > "$CERT_FILE_PUB"
     chmod 644 "$CERT_FILE_PUB"
 fi
 
+# Support variable set by go-c8y-cli
+if [ -n "$C8Y_DOMAIN" ] && [ -z "${TEDGE_C8Y_URL:-}" ]; then
+    echo "Setting c8y.url from C8Y_DOMAIN env variable. $C8Y_DOMAIN" >&2
+    tedge config set c8y.url "$C8Y_DOMAIN"
+fi
+
 #
 # Connect the mappers (if they are configured and not already connected)
 #
 MAPPERS="c8y az aws"
 for MAPPER in $MAPPERS; do
-    URL=$(tedge config get "${MAPPER}.url" 2>/dev/null)
-    if [ -n "$URL" ]; then
-        if ! tedge connect "$MAPPER" --test 2>/dev/null; then
-            echo "Connecting $MAPPER" >&2
-            tedge reconnect "$MAPPER" ||:
-        fi
+    if tedge config get "${MAPPER}.url" 2>/dev/null; then
+        tedge connect "$MAPPER" ||:
     fi
 done

docker-compose.yaml

@@ -4,7 +4,24 @@ services:
       dockerfile: Dockerfile
       context: .
     restart: always
+    user: "${DOCKER_USER:-tedge:tedge}"
+    secrets:
+      - source: certificate_private_key
+        mode: 0644
+        uid: "tedge"
+        gid: "tedge"
+      - source: certificate_public_key
+        mode: 0644
+        uid: "tedge"
+        gid: "tedge"
     env_file:
       - .env
     tmpfs:
       - /tmp
+
+secrets:
+  certificate_private_key:
+    file: device-certs/tedge-private-key.pem
+
+  certificate_public_key:
+    file: device-certs/tedge-certificate.pem

files/tedge/plugins/tedge-log-plugin.toml

@@ -1,4 +1,4 @@
 files = [
-    { type = "software-management", path = "/var/log/tedge/agent/software-*" },
+    { type = "software-management", path = "/var/log/tedge/agent/workflow-software*" },
     { type = "shell", path = "/var/log/tedge/agent/c8y_Command-*" },
 ]