Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add blacklisting support for ROS 1 / Add options to continue despite errors / Allow to build to build stage #25

Open
wants to merge 16 commits into
base: main
Choose a base branch
from
Open

Add blacklisting support for ROS 1 / Add options to continue despite errors / Allow to build to build stage #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
e78a7c9
warn that blacklisting is only supported for ros2
lreiher Jun 6, 2024
c759b4b
update readme notice
lreiher Jun 6, 2024
3954a36
implement blacklisting for ros1
lreiher Jun 6, 2024
754c31b
temporarily add printing of package removal script
lreiher Jun 6, 2024
b803b51
try to fix ros1 blacklisting
lreiher Jun 6, 2024
34c1e48
remove temporary print of pkg removal script
lreiher Jun 6, 2024
cab9bd1
remove warning that blacklisting is only supported for ros2
lreiher Jun 6, 2024
bc1309f
add options to toggle to continue rosdep and catkin/colcon on failure
lreiher Jun 6, 2024
20a8991
fix typo in .gitlab-ci
lreiher Jun 6, 2024
b5cd5ff
add option to build to devel space in ros1
lreiher Jun 6, 2024
bf8b8ce
fix missing github action inputs for new config vars
lreiher Jun 6, 2024
f1d405c
ignore failures in catkin/colcon build, if arg is set
lreiher Jun 6, 2024
e8e6b17
add option to build target=build image containing devel/install and s…
lreiher Jun 6, 2024
b77190f
fix dependencies of gitlab build jobs
lreiher Jun 6, 2024
3a1c1b4
only start run stage after build stage, if present
lreiher Jun 6, 2024
da7da2b
Merge branch 'main' into readme-updates
lreiher Nov 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
175 changes: 123 additions & 52 deletions .gitlab-ci/docker-ros.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,69 +9,81 @@ workflow:


variables:
TARGET: run # Target stage of Dockerfile (comma-separated list) [dev|run]
PLATFORM: amd64 # Target platform architecture (comma-separated list) [amd64|arm64|...]
BASE_IMAGE: '' # Base image name:tag (required)
COMMAND: '' # Launch command of run image (required if target=run)
IMAGE_NAME: ${CI_REGISTRY_IMAGE} # Image name of run image
IMAGE_TAG: latest # Image tag of run image
DEV_IMAGE_NAME: ${IMAGE_NAME} # Image name of dev image
DEV_IMAGE_TAG: ${IMAGE_TAG}-dev # Image tag of dev image
SLIM_IMAGE_NAME: ${IMAGE_NAME} # Image name of slim run image
SLIM_IMAGE_TAG: ${IMAGE_TAG}-slim # Image tag of slim run image
BUILD_CONTEXT: . # Build context of Docker build process
REGISTRY: ${CI_REGISTRY} # Docker registry to push images to
REGISTRY_USER: ${CI_REGISTRY_USER} # Docker registry username
REGISTRY_PASSWORD: ${CI_REGISTRY_PASSWORD} # Docker registry password
ENABLE_INDUSTRIAL_CI: 'false' # Enable industrial_ci
ENABLE_SLIM: 'true' # Enable an extra slimmed run image via slim (only if run stage is targeted)
SLIM_BUILD_ARGS: '--sensor-ipc-mode proxy --continue-after=10 --show-clogs --http-probe=false' # Arguments to `slim build` (except for `--target` and `--tag`)
ENABLE_SINGLEARCH_PUSH: 'false' # Enable push of single arch images with [-amd64|-arm64] postfix
ENABLE_PUSH_AS_LATEST: 'false' # Push images with tag `latest`/`latest-dev` in addition to the configured image names
RMW_IMPLEMENTATION: 'rmw_cyclonedds_cpp' # RMW implementation to use (only for ROS 2)
ROS_DISTRO: '' # ROS Distro (required if ROS is not installed in `base-image`)
DISABLE_ROS_INSTALLATION: 'false' # Disable automatic installation of `ros-$ROS_DISTRO-ros-core` package, e.g., if ROS is already installed in `base-image` and package is not available for the OS
GIT_HTTPS_SERVER: ${CI_SERVER_HOST} # Server URL (without protocol) for cloning private Git repositories via HTTPS
GIT_HTTPS_USER: gitlab-ci-token # Username for cloning private Git repositories via HTTPS
GIT_HTTPS_PASSWORD: ${CI_JOB_TOKEN} # Password for cloning private Git repositories via HTTPS
GIT_SSH_PRIVATE_KEY: '' # SSH private key for cloning private Git repositories via SSH
GIT_SSH_KNOWN_HOST_KEYS: '' # Known SSH host keys for cloning private Git repositories via SSH (may be obtained using `ssh-keyscan`)
ADDITIONAL_DEBS_FILE: docker/additional-debs.txt # Relative filepath to file containing additional apt deb packages to install
ENABLE_RECURSIVE_ADDITIONAL_DEBS: 'false' # Enable recursive discovery of files named `additional-debs-file`
ADDITIONAL_FILES_DIR: docker/additional-files # Relative path to directory containing additional files to copy into image"
ADDITIONAL_PIP_FILE: docker/additional-pip-requirements.txt # Relative filepath to file containing additional pip packages to install
ENABLE_RECURSIVE_ADDITIONAL_PIP: 'false' # Enable recursive discovery of files named `additional-pip-file`
BLACKLISTED_PACKAGES_FILE: docker/blacklisted-packages.txt # Relative filepath to file containing the blacklisted packages
ENABLE_RECURSIVE_BLACKLISTED_PACKAGES: 'false' # Enable recursive discovery of files named `blacklisted-packages-file`
CUSTOM_SCRIPT_FILE: docker/custom.sh # Relative filepath to script containing custom installation commands
ENABLE_RECURSIVE_CUSTOM_SCRIPT: 'false' # Enable recursive discovery of files named `custom-script-file`
VCS_IMPORT_FILE: .repos # Relative filepath to file containing additional repos to install via vcstools (only relevant if ENABLE_RECURSIVE_VCS_IMPORT=false)
ENABLE_RECURSIVE_VCS_IMPORT: 'true' # Enable recursive discovery of files named `*.repos`
ADDITIONAL_DEBS_FILE: docker/additional-debs.txt # Relative filepath to file containing additional apt deb packages to install
ADDITIONAL_FILES_DIR: docker/additional-files # Relative path to directory containing additional files to copy into image"
ADDITIONAL_PIP_FILE: docker/additional-pip-requirements.txt # Relative filepath to file containing additional pip packages to install
BASE_IMAGE: '' # Base image name:tag (required)
BLACKLISTED_PACKAGES_FILE: docker/blacklisted-packages.txt # Relative filepath to file containing the blacklisted packages
BUILD_CONTEXT: . # Build context of Docker build process
BUILD_IMAGE_NAME: ${IMAGE_NAME} # Image name of build image
BUILD_IMAGE_TAG: ${IMAGE_TAG}-build # Image tag of build image
COMMAND: '' # Launch command of build/run image (required if target=run)
CUSTOM_SCRIPT_FILE: docker/custom.sh # Relative filepath to script containing custom installation commands
DEV_IMAGE_NAME: ${IMAGE_NAME} # Image name of dev image
DEV_IMAGE_TAG: ${IMAGE_TAG}-dev # Image tag of dev image
DISABLE_ROS_INSTALLATION: 'false' # Disable automatic installation of `ros-$ROS_DISTRO-ros-core` package, e.g., if ROS is already installed in `base-image` and package is not available for the OS
ENABLE_CONTINUE_BUILD_DESPITE_ERRORS: 'false' # Enable `catkin build --continue-on-failure` / `colcon build --continue-on-error`
ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS: 'false' # Enable `rosdep install -r`
ENABLE_INDUSTRIAL_CI: 'false' # Enable industrial_ci
ENABLE_PUSH_AS_LATEST: 'false' # Push images with tag `latest`/`latest-dev` in addition to the configured image names
ENABLE_RECURSIVE_ADDITIONAL_DEBS: 'false' # Enable recursive discovery of files named `additional-debs-file`
ENABLE_RECURSIVE_ADDITIONAL_PIP: 'false' # Enable recursive discovery of files named `additional-pip-file`
ENABLE_RECURSIVE_BLACKLISTED_PACKAGES: 'false' # Enable recursive discovery of files named `blacklisted-packages-file`
ENABLE_RECURSIVE_CUSTOM_SCRIPT: 'false' # Enable recursive discovery of files named `custom-script-file`
ENABLE_RECURSIVE_VCS_IMPORT: 'true' # Enable recursive discovery of files named `*.repos`
ENABLE_ROS1_DEVEL_SPACE: 'false' # Enable building to ROS devel space instead of install space (ROS 1 only)
ENABLE_SINGLEARCH_PUSH: 'false' # Enable push of single arch images with [-amd64|-arm64] postfix
ENABLE_SLIM: 'true' # Enable an extra slimmed run image via slim (only if run stage is targeted)
GIT_HTTPS_PASSWORD: ${CI_JOB_TOKEN} # Password for cloning private Git repositories via HTTPS
GIT_HTTPS_SERVER: ${CI_SERVER_HOST} # Server URL (without protocol) for cloning private Git repositories via HTTPS
GIT_HTTPS_USER: gitlab-ci-token # Username for cloning private Git repositories via HTTPS
GIT_SSH_KNOWN_HOST_KEYS: '' # Known SSH host keys for cloning private Git repositories via SSH (may be obtained using `ssh-keyscan`)
GIT_SSH_PRIVATE_KEY: '' # SSH private key for cloning private Git repositories via SSH
IMAGE_NAME: ${CI_REGISTRY_IMAGE} # Image name of run image
IMAGE_TAG: latest # Image tag of run image
PLATFORM: amd64 # Target platform architecture (comma-separated list) [amd64|arm64|...]
REGISTRY_PASSWORD: ${CI_REGISTRY_PASSWORD} # Docker registry password
REGISTRY_USER: ${CI_REGISTRY_USER} # Docker registry username
REGISTRY: ${CI_REGISTRY} # Docker registry to push images to
RMW_IMPLEMENTATION: 'rmw_cyclonedds_cpp' # RMW implementation to use (only for ROS 2)
ROS_DISTRO: '' # ROS Distro (required if ROS is not installed in `base-image`)
SLIM_BUILD_ARGS: '--sensor-ipc-mode proxy --continue-after=10 --show-clogs --http-probe=false' # Arguments to `slim build` (except for `--target` and `--tag`)
SLIM_IMAGE_NAME: ${IMAGE_NAME} # Image name of slim run image
SLIM_IMAGE_TAG: ${IMAGE_TAG}-slim # Image tag of slim run image
TARGET: run # Target stage of Dockerfile (comma-separated list) [dev|build|run]
VCS_IMPORT_FILE: .repos # Relative filepath to file containing additional repos to install via vcstools (only relevant if ENABLE_RECURSIVE_VCS_IMPORT=false)
# -----
DOCKER_ROS_GIT_REF: main

_RUN_IMAGE: ${IMAGE_NAME}:${IMAGE_TAG}
_BUILD_IMAGE: ${BUILD_IMAGE_NAME}:${BUILD_IMAGE_TAG}
_DEV_IMAGE: ${DEV_IMAGE_NAME}:${DEV_IMAGE_TAG}
_SLIM_IMAGE: ${SLIM_IMAGE_NAME}:${SLIM_IMAGE_TAG}

_IMAGE_DEV_CI: ${_DEV_IMAGE}_${CI_COMMIT_REF_SLUG}_ci
_IMAGE_RUN_CI: ${_RUN_IMAGE}_${CI_COMMIT_REF_SLUG}_ci
_IMAGE_SLIM_CI: ${_SLIM_IMAGE}_${CI_COMMIT_REF_SLUG}_ci
_IMAGE_DEV_CI_AMD64: ${_IMAGE_DEV_CI}-amd64
_IMAGE_DEV_CI_ARM64: ${_IMAGE_DEV_CI}-arm64
_IMAGE_RUN_CI_AMD64: ${_IMAGE_RUN_CI}-amd64
_IMAGE_RUN_CI_ARM64: ${_IMAGE_RUN_CI}-arm64
_IMAGE_BUILD_CI_AMD64: ${_IMAGE_BUILD_CI}-amd64
_IMAGE_BUILD_CI_ARM64: ${_IMAGE_BUILD_CI}-arm64
_IMAGE_BUILD_CI: ${_BUILD_IMAGE}_${CI_COMMIT_REF_SLUG}_ci
_IMAGE_BUILD_LATEST: ${BUILD_IMAGE_NAME}:latest-build
_IMAGE_BUILD_TAG: ${BUILD_IMAGE_NAME}:${CI_COMMIT_TAG}-build
_IMAGE_BUILD_TARGET_TAG: ${_BUILD_IMAGE}-${CI_COMMIT_TAG}
_IMAGE_DEV_CI_AMD64: ${_IMAGE_DEV_CI}-amd64
_IMAGE_DEV_CI_ARM64: ${_IMAGE_DEV_CI}-arm64
_IMAGE_DEV_CI: ${_DEV_IMAGE}_${CI_COMMIT_REF_SLUG}_ci
_IMAGE_DEV_LATEST: ${DEV_IMAGE_NAME}:latest-dev
_IMAGE_DEV_TAG: ${DEV_IMAGE_NAME}:${CI_COMMIT_TAG}-dev
_IMAGE_DEV_TARGET_TAG: ${_DEV_IMAGE}-${CI_COMMIT_TAG}
_IMAGE_RUN_CI_AMD64: ${_IMAGE_RUN_CI}-amd64
_IMAGE_RUN_CI_ARM64: ${_IMAGE_RUN_CI}-arm64
_IMAGE_RUN_CI: ${_RUN_IMAGE}_${CI_COMMIT_REF_SLUG}_ci
_IMAGE_RUN_LATEST: ${IMAGE_NAME}:latest
_IMAGE_RUN_TAG: ${IMAGE_NAME}:${CI_COMMIT_TAG}
_IMAGE_RUN_TARGET_TAG: ${_RUN_IMAGE}-${CI_COMMIT_TAG}
_IMAGE_SLIM_CI_AMD64: ${_IMAGE_SLIM_CI}-amd64
_IMAGE_SLIM_CI_ARM64: ${_IMAGE_SLIM_CI}-arm64
_IMAGE_DEV_LATEST: ${DEV_IMAGE_NAME}:latest-dev
_IMAGE_RUN_LATEST: ${IMAGE_NAME}:latest
_IMAGE_SLIM_CI: ${_SLIM_IMAGE}_${CI_COMMIT_REF_SLUG}_ci
_IMAGE_SLIM_LATEST: ${SLIM_IMAGE_NAME}:latest-slim
_IMAGE_DEV_TARGET_TAG: ${_DEV_IMAGE}-${CI_COMMIT_TAG}
_IMAGE_RUN_TARGET_TAG: ${_RUN_IMAGE}-${CI_COMMIT_TAG}
_IMAGE_SLIM_TARGET_TAG: ${_SLIM_IMAGE}-${CI_COMMIT_TAG}
_IMAGE_DEV_TAG: ${DEV_IMAGE_NAME}:${CI_COMMIT_TAG}-dev
_IMAGE_RUN_TAG: ${IMAGE_NAME}:${CI_COMMIT_TAG}
_IMAGE_SLIM_TAG: ${SLIM_IMAGE_NAME}:${CI_COMMIT_TAG}-slim
_IMAGE_SLIM_TARGET_TAG: ${_SLIM_IMAGE}-${CI_COMMIT_TAG}

GIT_SUBMODULE_STRATEGY: recursive
DOCKER_DRIVER: overlay2
Expand All @@ -81,6 +93,7 @@ variables:

stages:
- Build dev Images
- Build build Images
- Build run Images
- Test ROS Industrial CI
- Slim Images
Expand Down Expand Up @@ -150,12 +163,45 @@ dev-arm64:
ENABLE_SLIM: 'false'
_IMAGE_POSTFIX: _${CI_COMMIT_REF_SLUG}_ci

build-amd64:
stage: Build build Images
extends: .build
needs:
- job: dev-amd64
optional: true
rules:
- if: $PLATFORM =~ /.*amd64.*/ && $TARGET =~ /.*build.*/
variables:
_PLATFORM: amd64
_TARGET: build
IMAGE: ${_IMAGE_BUILD_CI_AMD64}
ENABLE_SINGLEARCH_PUSH: 'true'
_IMAGE_POSTFIX: _${CI_COMMIT_REF_SLUG}_ci

build-arm64:
stage: Build build Images
extends: .build
tags: [privileged, arm64]
needs:
- job: dev-arm64
optional: true
rules:
- if: $PLATFORM =~ /.*arm64.*/ && $TARGET =~ /.*build.*/
variables:
_PLATFORM: arm64
_TARGET: build
IMAGE: ${_IMAGE_BUILD_CI_ARM64}
ENABLE_SINGLEARCH_PUSH: 'true'
_IMAGE_POSTFIX: _${CI_COMMIT_REF_SLUG}_ci

run-amd64:
stage: Build run Images
extends: .build
needs:
- job: dev-amd64
optional: true
- job: build-amd64
optional: true
rules:
- if: $PLATFORM =~ /.*amd64.*/ && $TARGET =~ /.*run.*/
variables:
Expand All @@ -173,6 +219,8 @@ run-arm64:
needs:
- job: dev-arm64
optional: true
- job: build-arm64
optional: true
rules:
- if: $PLATFORM =~ /.*arm64.*/ && $TARGET =~ /.*run.*/
variables:
Expand Down Expand Up @@ -300,6 +348,10 @@ Slim run-arm64:
optional: true
- job: dev-arm64
optional: true
- job: build-amd64
optional: true
- job: build-arm64
optional: true
- job: run-amd64
optional: true
- job: run-arm64
Expand All @@ -326,6 +378,10 @@ Slim run-arm64:
docker manifest create ${IMG_DEV} --amend ${_IMAGE_DEV_CI_AMD64} --amend ${_IMAGE_DEV_CI_ARM64}
docker manifest push ${IMG_DEV}
fi
if [[ "${TARGET}" =~ build ]]; then
docker manifest create ${IMG_BUILD} --amend ${_IMAGE_BUILD_CI_AMD64} --amend ${_IMAGE_BUILD_CI_ARM64}
docker manifest push ${IMG_BUILD}
fi
if [[ "${TARGET}" =~ run ]]; then
docker manifest create ${IMG_RUN} --amend ${_IMAGE_RUN_CI_AMD64} --amend ${_IMAGE_RUN_CI_ARM64}
docker manifest push ${IMG_RUN}
Expand All @@ -340,6 +396,11 @@ Slim run-arm64:
docker tag ${_IMAGE_DEV_CI_AMD64} ${IMG_DEV}
docker push ${IMG_DEV}
fi
if [[ "${TARGET}" =~ build ]]; then
docker pull ${_IMAGE_BUILD_CI_AMD64}
docker tag ${_IMAGE_BUILD_CI_AMD64} ${IMG_BUILD}
docker push ${IMG_BUILD}
fi
if [[ "${TARGET}" =~ run ]]; then
docker pull ${_IMAGE_RUN_CI_AMD64}
docker tag ${_IMAGE_RUN_CI_AMD64} ${IMG_RUN}
Expand All @@ -356,6 +417,11 @@ Slim run-arm64:
docker tag ${_IMAGE_DEV_CI_ARM64} ${IMG_DEV}
docker push ${IMG_DEV}
fi
if [[ "${TARGET}" =~ build ]]; then
docker pull ${_IMAGE_BUILD_CI_ARM64}
docker tag ${_IMAGE_BUILD_CI_ARM64} ${IMG_BUILD}
docker push ${IMG_BUILD}
fi
if [[ "${TARGET}" =~ run ]]; then
docker pull ${_IMAGE_RUN_CI_ARM64}
docker tag ${_IMAGE_RUN_CI_ARM64} ${IMG_RUN}
Expand All @@ -378,6 +444,7 @@ Push CI:
- if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
variables:
IMG_DEV: ${_IMAGE_DEV_CI}
IMG_BUILD: ${_IMAGE_BUILD_CI}
IMG_RUN: ${_IMAGE_RUN_CI}
IMG_SLIM: ${_IMAGE_SLIM_CI}

Expand All @@ -389,6 +456,7 @@ Push:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
variables:
IMG_DEV: ${_DEV_IMAGE}
IMG_BUILD: ${_BUILD_IMAGE}
IMG_RUN: ${_RUN_IMAGE}
IMG_SLIM: ${_SLIM_IMAGE}

Expand All @@ -400,6 +468,7 @@ Push latest:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $ENABLE_PUSH_AS_LATEST == 'true'
variables:
IMG_DEV: ${_IMAGE_DEV_LATEST}
IMG_BUILD: ${_IMAGE_BUILD_LATEST}
IMG_RUN: ${_IMAGE_RUN_LATEST}
IMG_SLIM: ${_IMAGE_SLIM_LATEST}

Expand All @@ -411,6 +480,7 @@ Push target tag:
- if: $CI_COMMIT_TAG
variables:
IMG_DEV: ${_IMAGE_DEV_TARGET_TAG}
IMG_BUILD: ${_IMAGE_BUILD_TARGET_TAG}
IMG_RUN: ${_IMAGE_RUN_TARGET_TAG}
IMG_SLIM: ${_IMAGE_SLIM_TARGET_TAG}

Expand All @@ -422,5 +492,6 @@ Push tag:
- if: $CI_COMMIT_TAG && $ENABLE_PUSH_AS_LATEST == 'true'
variables:
IMG_DEV: ${_IMAGE_DEV_TAG}
IMG_BUILD: ${_IMAGE_BUILD_TAG}
IMG_RUN: ${_IMAGE_RUN_TAG}
IMG_SLIM: ${_IMAGE_SLIM_TAG}
Loading
Loading