ci: build targets using px4-dev container

.github/workflows/build_all_targets.yml

@@ -26,7 +26,6 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
       timestamp: ${{ steps.set-timestamp.outputs.timestamp }}
-      tagname: ${{ steps.set-tag.outputs.tagname }}
       branchname: ${{ steps.set-branch.outputs.branchname }}
     steps:
       - uses: actions/checkout@v4
@@ -57,9 +56,9 @@ jobs:
           echo "$(./Tools/ci/generate_board_targets_json.py --group --verbose)"
 
   setup:
-    name: Build Group [${{ matrix.group }}]
+    name: Build Group [${{ matrix.group }}][${{ matrix.arch == 'nuttx' && 'x86' || 'arm64' }}]
     # runs-on: ubuntu-latest
-    runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false]
+    runs-on: [runs-on,"runner=8cpu-linux-${{ matrix.arch == 'nuttx' && 'x64' || 'arm64' }}","image=ubuntu24-full-${{ matrix.arch == 'nuttx' && 'x64' || 'arm64' }}","run-id=${{ github.run_id }}",spot=false]
     needs: group_targets
     strategy:
       matrix: ${{ fromJson(needs.group_targets.outputs.matrix) }}
@@ -94,7 +93,7 @@ jobs:
 
       - name: Building [${{ matrix.group }}]
         run: |
-            ./Tools/ci/build_all_runner.sh ${{matrix.targets}}
+            ./Tools/ci/build_all_runner.sh ${{matrix.targets}} ${{matrix.arch}}
 
       - name: Arrange Build Artifacts
         run: |
@@ -112,7 +111,7 @@ jobs:
   artifacts:
     name: Upload Artifacts to S3
     # runs-on: ubuntu-latest
-    runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false]
+    runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
     needs: [setup, group_targets]
     if: contains(fromJSON('["main", "stable", "beta"]'), needs.group_targets.outputs.branchname)
     steps:
@@ -140,10 +139,12 @@ jobs:
 
   release:
     name: Create Release and Upload Artifacts
+    permissions:
+      contents: write
     # runs-on: ubuntu-latest
-    runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false]
+    runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
     needs: [setup, group_targets]
-    if: startsWith(github.ref, 'refs/tags/v')
+    if: startsWith(github.ref, 'refs/tags/')
     steps:
       - name: Download Artifacts
         uses: actions/download-artifact@v4

.github/workflows/dev_container.yml

@@ -16,43 +16,58 @@ on:
 
 jobs:
   build:
-    name: Build and Push Container
-    runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
+    name: Build and Push Container (${{ matrix.arch == 'x64' && 'x86' || 'arm64' }})
+    strategy:
+      matrix:
+        arch: [x64, arm64]
+    runs-on: [runs-on,"runner=8cpu-linux-${{ matrix.arch }}","image=ubuntu24-full-${{ matrix.arch }}","run-id=${{ github.run_id }}",spot=false,extras=s3-cache]
+    outputs:
+      px4_version: ${{ steps.px4_version.outputs.px4_version }}
+    #   tags: ${{ steps.meta.outputs.tags }}
+    #   labels: ${{ steps.meta.outputs.labels }}
+    #   metadata: ${{ steps.docker.outputs.metadata }}
+      image_name: ${{ steps.image_name.outputs.image_name }}
     steps:
+      - uses: runs-on/action@v1
       - uses: actions/checkout@v4
         with:
           fetch-tags: true
           submodules: false
           fetch-depth: 0
 
-      - name: Set PX4 Tag
-        id: px4-tag
+      - name: Set PX4 Tag Version
+        id: px4_version
         run: |
-          echo "tag=$(git describe --tags --match 'v[0-9]*')" >> $GITHUB_OUTPUT
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        if: github.event_name != 'pull_request'
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          echo "px4_version=$(git describe --tags --match 'v[0-9]*')" >> $GITHUB_OUTPUT
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
         uses: docker/metadata-action@v5
         with:
           images: |
             ghcr.io/PX4/px4-dev
-            ${{ (github.event_name != 'pull_request') && 'px4io/px4-dev' || '' }}
+            px4io/px4-dev
           tags: |
-            type=raw,enable=true,value=${{ steps.px4-tag.outputs.tag }},priority=1000
+            type=raw,enable=true,value=${{ steps.px4_version.outputs.px4_version }},priority=1000
+
+      - name: Cache Apt Registry
+        uses: actions/cache@v3
+        id: cache
+        with:
+          path: |
+            var-cache-apt
+            var-lib-apt
+          key: cache-${{ hashFiles('Tools/setup/Dockerfile') }}
+
+      - name: Inject Cache into Docker
+        uses: reproducible-containers/[email protected]
+        with:
+          cache-map: |
+            {
+              "var-cache-apt": "/var/cache/apt",
+              "var-lib-apt": "/var/lib/apt"
+            }
+          skip-extraction: ${{ steps.cache.outputs.cache-hit }}
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -64,38 +79,92 @@ jobs:
           context: Tools/setup
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          platforms: |
-            linux/amd64
+          platforms: linux/${{ matrix.arch == 'x64' && 'amd64' || 'arm64' }}
           load: true
           push: false
-          cache-from: type=s3,blobs_prefix=cache/${{ github.repository }}/,manifests_prefix=cache/${{ github.repository }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
-          cache-to: type=s3,blobs_prefix=cache/${{ github.repository }}/,manifests_prefix=cache/${{ github.repository }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
-      - name: Get Tag Name
-        id: tag_name
+      - name: Get Container Image Tag Name
+        id: image_name
         run: |
-          echo "::set-output name=tag_name::$(echo '${{ fromJSON(steps.docker.outputs.metadata)['image.name'] }}' | sed 's/,.*$//')"
+          echo "image_name=$(echo '${{ fromJSON(steps.docker.outputs.metadata)['image.name'] }}' | sed 's/,.*$//')" >> $GITHUB_OUTPUT
 
       - name: make quick_check
         uses: addnab/docker-run-action@v3
         with:
-          image: ${{ steps.tag_name.outputs.tag_name }}
+          image: ${{ steps.image_name.outputs.image_name }}
           options: -v ${{ github.workspace }}:/workspace
           run: |
             cd /workspace
             git config --global --add safe.directory /workspace
             make px4_sitl_default
             make px4_fmu-v6x_default
 
-      - name: Push container image
+      - name: Build and export
         uses: docker/build-push-action@v6
         with:
-          context: Tools/setup
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          platforms: |
-            linux/amd64
-          provenance: mode=max
-          push: ${{ github.event_name != 'pull_request' }}
-          cache-from: type=s3,blobs_prefix=cache/${{ github.repository }}/,manifests_prefix=cache/${{ github.repository }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
-          cache-to: type=s3,blobs_prefix=cache/${{ github.repository }}/,manifests_prefix=cache/${{ github.repository }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+          tags: ${{ steps.image_name.outputs.image_name }}
+          outputs: type=docker,dest=${{ runner.temp }}/px4-builder-image-${{ matrix.arch }}.tar
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.image_name.outputs.image_name }}-${{ matrix.arch }}
+          path: ${{ runner.temp }}/px4-builder-image-${{ matrix.arch }}.tar
+
+
+  push:
+    runs-on: [runs-on,"runner=8cpu-linux-x64","image=ubuntu24-full-x64","run-id=${{ github.run_id }}",spot=false,extras=s3-cache]
+    needs: build
+    steps:
+      - uses: runs-on/action@v1
+      - uses: actions/checkout@v4
+        with:
+          fetch-tags: true
+          submodules: false
+          fetch-depth: 0
+
+      - name: Download x64 Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ steps.image_name.outputs.image_name }}-x64
+          path: ${{ runner.temp }}
+
+      - name: Download x86 Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ steps.image_name.outputs.image_name }}-x86
+          path: ${{ runner.temp }}
+
+      - name: Setup tmate session
+        # if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
+
+      # - name: Login to Docker Hub
+      #   uses: docker/login-action@v3
+      #   # if: ${{ startsWith(github.ref, 'refs/tags/') }}
+      #   with:
+      #     username: ${{ secrets.DOCKERHUB_USERNAME }}
+      #     password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      # - name: Login to GitHub Container Registry
+      #   uses: docker/login-action@v3
+      #   # if: ${{ startsWith(github.ref, 'refs/tags/') }}
+      #   with:
+      #     registry: ghcr.io
+      #     username: ${{ github.actor }}
+      #     password: ${{ secrets.GITHUB_TOKEN }}
+
+      # - name: Push container image
+      #   uses: docker/build-push-action@v6
+      #   # if: ${{ startsWith(github.ref, 'refs/tags/') }}
+      #   with:
+      #     context: Tools/setup
+      #     tags: ${{ needs.build.outputs.tags }}
+      #     labels: ${{ needs.build.outputs.labels }}
+      #     platforms: linux/amd64,linux/arm64
+      #     provenance: mode=max
+      #     push: true
+      #     cache-from: type=gha
+      #     cache-to: type=gha,mode=max

Tools/ci/generate_board_targets_json.py

@@ -87,20 +87,17 @@ def process_target(px4board_file, target_name):
 
     if platform not in excluded_platforms:
         # get the container based on the platform and toolchain
+        container = 'ghcr.io/px4/px4-dev:v1.16.0-alpha2-388-ga996439b1d'
         if platform == 'posix':
-            container = 'px4io/px4-dev-base-focal:2021-09-08'
             group = 'base'
             if toolchain:
                 if toolchain.startswith('aarch64'):
-                    container = 'px4io/px4-dev-aarch64:2022-08-12'
                     group = 'aarch64'
                 elif toolchain == 'arm-linux-gnueabihf':
-                    container = 'px4io/px4-dev-armhf:2023-06-26'
                     group = 'armhf'
                 else:
                     if verbose: print(f'unmatched toolchain: {toolchain}')
         elif platform == 'nuttx':
-            container = 'px4io/px4-dev-nuttx-focal:2022-08-12'
             group = 'nuttx'
         else:
             if verbose: print(f'unmatched platform: {platform}')
@@ -124,7 +121,7 @@ def process_target(px4board_file, target_name):
 # - Events
 metadata_targets = ['airframe_metadata', 'parameters_metadata', 'extract_events']
 grouped_targets['base'] = {}
-grouped_targets['base']['container'] = 'px4io/px4-dev-base-focal:2021-09-08'
+grouped_targets['base']['container'] = 'ghcr.io/px4/px4-dev:v1.16.0-alpha2-388-ga996439b1d'
 grouped_targets['base']['manufacturers'] = {}
 grouped_targets['base']['manufacturers']['px4'] = []
 grouped_targets['base']['manufacturers']['px4'] += metadata_targets

Tools/ci/package_build_artifacts.sh

@@ -4,8 +4,9 @@ mkdir artifacts
 cp **/**/*.px4 artifacts/
 cp **/**/*.elf artifacts/
 for build_dir_path in build/*/ ; do
+  build_dir_path=${build_dir_path::${#build_dir_path}-1}
   build_dir=${build_dir_path#*/}
-  build_dir=${build_dir::${#build_dir}-1}
+  target_name=$build_dir
   mkdir artifacts/$build_dir
   find artifacts/ -maxdepth 1 -type f -name "*$build_dir*"
   # Airframe
@@ -26,21 +27,23 @@ for build_dir_path in build/*/ ; do
   echo "----------"
 done
 
-# general metadata
-mkdir artifacts/_general/
-cp artifacts/px4_sitl_default/airframes.xml artifacts/_general/
-# Airframe
-cp artifacts/px4_sitl_default/airframes.xml artifacts/_general/
-# Parameters
-cp artifacts/px4_sitl_default/parameters.xml artifacts/_general/
-cp artifacts/px4_sitl_default/parameters.json artifacts/_general/
-cp artifacts/px4_sitl_default/parameters.json.xz artifacts/_general/
-# Actuators
-cp artifacts/px4_sitl_default/actuators.json artifacts/_general/
-cp artifacts/px4_sitl_default/actuators.json.xz artifacts/_general/
-# Events
-cp artifacts/px4_sitl_default/events/all_events.json.xz artifacts/_general/
-# ROS 2 msgs
-cp artifacts/px4_sitl_default/events/all_events.json.xz artifacts/_general/
-# Module Docs
-ls -la artifacts/_general/
+if [ -d artifacts/px4_sitl_default ]; then
+  # general metadata
+  mkdir artifacts/_general/
+  cp artifacts/px4_sitl_default/airframes.xml artifacts/_general/
+  # Airframe
+  cp artifacts/px4_sitl_default/airframes.xml artifacts/_general/
+  # Parameters
+  cp artifacts/px4_sitl_default/parameters.xml artifacts/_general/
+  cp artifacts/px4_sitl_default/parameters.json artifacts/_general/
+  cp artifacts/px4_sitl_default/parameters.json.xz artifacts/_general/
+  # Actuators
+  cp artifacts/px4_sitl_default/actuators.json artifacts/_general/
+  cp artifacts/px4_sitl_default/actuators.json.xz artifacts/_general/
+  # Events
+  cp artifacts/px4_sitl_default/events/all_events.json.xz artifacts/_general/
+  # ROS 2 msgs
+  cp artifacts/px4_sitl_default/events/all_events.json.xz artifacts/_general/
+  # Module Docs
+  ls -la artifacts/_general/
+fi

Tools/setup/Dockerfile

@@ -13,6 +13,14 @@ ENV TZ=UTC
 EXPOSE 14556/udp
 EXPOSE 14557/udp
 
+# Set Up Caching of APT packages
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+	--mount=type=cache,target=/var/lib/apt,sharing=locked \
+	rm -f /etc/apt/apt.conf.d/docker-clean && \
+	echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >/etc/apt/apt.conf.d/keep-cache && \
+	apt-get --quiet -y update
+
+# Install Shell Script Entrypoint
 COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
 # Install PX4 Requirements

Tools/setup/docker-entrypoint.sh

@@ -1,19 +1,27 @@
 #!/bin/bash
 
+GREEN='\033[0;32m'
+NC='\033[0m' # No Color
+FILE_DESCRIPTOR="${GREEN}[docker-entrypoint.sh]${NC}"
+
+echo -e "$FILE_DESCRIPTOR Starting"
+
 # Start virtual X server in the background
 # - DISPLAY default is :99, set in dockerfile
 # - Users can override with `-e DISPLAY=` in `docker run` command to avoid
 #   running Xvfb and attach their screen
 if [[ -x "$(command -v Xvfb)" && "$DISPLAY" == ":99" ]]; then
-	echo "[docker-entrypoint.sh] Starting Xvfb"
+	echo -e "$FILE_DESCRIPTOR Starting Xvfb"
 	Xvfb :99 -screen 0 1600x1200x24+32 &
 fi
 
 # Check if the ROS_DISTRO is passed and use it
 # to source the ROS environment
 if [ -n "${ROS_DISTRO}" ]; then
-	echo "[docker-entrypoint.sh] ROS: ${ROS_DISTRO}"
+	echo -e "$FILE_DESCRIPTOR ROS: ${ROS_DISTRO}"
 	source "/opt/ros/$ROS_DISTRO/setup.bash"
 fi
 
+echo -e "$FILE_DESCRIPTOR ($( uname -m ))"
+
 exec "$@"