From eaed01929e215390b7781129c74ac3c5c47bd183 Mon Sep 17 00:00:00 2001 From: Chris Lalancette Date: Thu, 3 Oct 2024 15:13:35 -0400 Subject: [PATCH 1/5] Switch to using Pixi/Conda for Windows. This allows us to much more easily manage dependencies, and to update them in the future. We accomplish this by first, removing the chef cookbooks. Next, we rewrite the Windows dockerfile to install MSVC, install Connext, install pixi, fetch the dependency list from https://github.com/ros2/ros2 , install those dependencies, and then run the build. Signed-off-by: Chris Lalancette --- .gitmodules | 7 - job_templates/ci_job.xml.em | 12 -- job_templates/packaging_job.xml.em | 12 -- ros2_batch_job/__main__.py | 128 +++--------------- ros2_batch_job/linux_batch/__init__.py | 2 +- ros2_batch_job/windows_batch/__init__.py | 2 +- windows_docker_resources/Dockerfile | 75 +++++----- windows_docker_resources/README.md | 73 +--------- .../install_ros2_humble.json | 25 ---- .../install_ros2_iron.json | 25 ---- .../install_ros2_jazzy.json | 25 ---- .../install_ros2_rolling.json | 25 ---- windows_docker_resources/qtaccount | 1 - windows_docker_resources/ros2-cookbooks | 1 - windows_docker_resources/solo.rb | 3 - 15 files changed, 60 insertions(+), 356 deletions(-) delete mode 100644 windows_docker_resources/install_ros2_humble.json delete mode 100644 windows_docker_resources/install_ros2_iron.json delete mode 100644 windows_docker_resources/install_ros2_jazzy.json delete mode 100644 windows_docker_resources/install_ros2_rolling.json delete mode 160000 windows_docker_resources/qtaccount delete mode 160000 windows_docker_resources/ros2-cookbooks delete mode 100644 windows_docker_resources/solo.rb diff --git a/.gitmodules b/.gitmodules index d13fd6b59..4aefa7085 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,10 +14,3 @@ path = windows_docker_resources/rticonnextdds-license url = git@github.com:osrf/rticonnextdds-license.git branch = license -[submodule "windows_docker_resources/qtaccount"] - path = windows_docker_resources/qtaccount - url = git@github.com:osrf/qtaccount -[submodule "windows_docker_resources/ros2-cookbooks"] - path = windows_docker_resources/ros2-cookbooks - url = git@github.com:ros-infrastructure/ros2-cookbooks - branch = latest diff --git a/job_templates/ci_job.xml.em b/job_templates/ci_job.xml.em index 75d0a29bb..1b4a20361 100644 --- a/job_templates/ci_job.xml.em +++ b/job_templates/ci_job.xml.em @@ -173,7 +173,6 @@ if [ "$CI_ENABLE_COVERAGE" = "true" ]; then fi if [ -n "${CI_ROS_DISTRO+x}" ]; then export DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS} --build-arg ROS_DISTRO=${CI_ROS_DISTRO}" - export CI_ARGS="$CI_ARGS --ros-distro $CI_ROS_DISTRO" fi if [ -n "${CI_BUILD_ARGS+x}" ]; then export CI_ARGS="$CI_ARGS --build-args $CI_BUILD_ARGS" @@ -242,14 +241,6 @@ rmdir /S /Q ws workspace "work space" echo "# BEGIN SECTION: Build DockerFile" set CONTAINER_NAME=ros2_windows_ci_%CI_ROS_DISTRO% set DOCKERFILE=windows_docker_resources\Dockerfile -set SOLO_FILE=windows_docker_resources\install_ros2_%CI_ROS_DISTRO%.json -set VISUAL_STUDIO_VERSION=%CI_VISUAL_STUDIO_VERSION% - -rem "Change dockerfile once per day to invalidate docker caches" -powershell "(Get-Content ${Env:DOCKERFILE}).replace('@@todays_date', $(Get-Date).ToLongDateString()) | Set-Content ${Env:DOCKERFILE}" - -rem "Change the chef-solo configuration file to set a specific Visual Studio version. -powershell -noexit "(Get-Content ${Env:SOLO_FILE}).replace('@@vs_version', ${Env:VISUAL_STUDIO_VERSION}) | Set-Content ${Env:SOLO_FILE}" rem "Finding the Release Version is much easier with powershell than cmd" powershell $(Get-ItemProperty -Path 'HKLM:SOFTWARE\Microsoft\Windows NT\CurrentVersion\Update\TargetingInfo\Installed\Server.OS.amd64' -Name Version).Version > release_version.txt @@ -267,9 +258,6 @@ if "!CI_BRANCH_TO_TEST!" NEQ "" ( if "!CI_COLCON_BRANCH!" NEQ "" ( set "CI_ARGS=!CI_ARGS! --colcon-branch !CI_COLCON_BRANCH!" ) -if "!CI_ROS_DISTRO!" NEQ "" ( - set "CI_ARGS=!CI_ARGS! --ros-distro !CI_ROS_DISTRO!" -) if "!CI_USE_WHITESPACE_IN_PATHS!" == "true" ( set "CI_ARGS=!CI_ARGS! --white-space-in sourcespace buildspace installspace workspace" ) diff --git a/job_templates/packaging_job.xml.em b/job_templates/packaging_job.xml.em index ddc1dd88b..c68f52832 100644 --- a/job_templates/packaging_job.xml.em +++ b/job_templates/packaging_job.xml.em @@ -164,7 +164,6 @@ if [ "$CI_ISOLATED" = "true" ]; then fi if [ -n "${CI_ROS_DISTRO+x}" ]; then export DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS} --build-arg ROS_DISTRO=${CI_ROS_DISTRO}" - export CI_ARGS="$CI_ARGS --ros-distro $CI_ROS_DISTRO" fi if [ -n "${CI_COLCON_MIXIN_URL+x}" ]; then export CI_ARGS="$CI_ARGS --colcon-mixin-url $CI_COLCON_MIXIN_URL" @@ -238,14 +237,6 @@ rmdir /S /Q ws workspace set CONTAINER_NAME=ros2_windows_ci_%CI_ROS_DISTRO% set DOCKERFILE=windows_docker_resources\Dockerfile -set SOLO_FILE=windows_docker_resources\install_ros2_%CI_ROS_DISTRO%.json -set VISUAL_STUDIO_VERSION=%CI_VISUAL_STUDIO_VERSION% - -rem "Change dockerfile once per day to invalidate docker caches" -powershell "(Get-Content ${Env:DOCKERFILE}).replace('@@todays_date', $(Get-Date).ToLongDateString()) | Set-Content ${Env:DOCKERFILE}" - -rem "Change the chef-solo configuration file to set a specific Visual Studio version. -powershell -noexit "(Get-Content ${Env:SOLO_FILE}).replace('@@vs_version', ${Env:VISUAL_STUDIO_VERSION}) | Set-Content ${Env:SOLO_FILE}" rem "Finding the Release Version is much easier with powershell than cmd" powershell $(Get-ItemProperty -Path 'HKLM:SOFTWARE\Microsoft\Windows NT\CurrentVersion\Update\TargetingInfo\Installed\Server.OS.amd64' -Name Version).Version > release_version.txt @@ -263,9 +254,6 @@ if "!CI_BRANCH_TO_TEST!" NEQ "" ( if "!CI_COLCON_BRANCH!" NEQ "" ( set "CI_ARGS=!CI_ARGS! --colcon-branch !CI_COLCON_BRANCH!" ) -if "!CI_ROS_DISTRO!" NEQ "" ( - set "CI_ARGS=!CI_ARGS! --ros-distro !CI_ROS_DISTRO!" -) if "!CI_USE_CONNEXTDDS!" == "false" ( set "CI_ARGS=!CI_ARGS! --ignore-rmw rmw_connextdds" ) diff --git a/ros2_batch_job/__main__.py b/ros2_batch_job/__main__.py index 5c5e7dad0..b026b89f3 100644 --- a/ros2_batch_job/__main__.py +++ b/ros2_batch_job/__main__.py @@ -50,42 +50,6 @@ PipPackage = collections.namedtuple('PipPackage', ['pkgname' ,'importname', 'version']) -pip_dependencies = [ - PipPackage('EmPy', 'em', '<4'), - PipPackage('catkin_pkg', 'catkin_pkg', ''), - PipPackage('cryptography', 'cryptography', '<=3.0'), - PipPackage('coverage', 'coverage', ''), - PipPackage('flake8', 'flake8', '<5.0.0'), - PipPackage('flake8-blind-except', 'flake8_blind_except', '==0.1.1'), - PipPackage('flake8-builtins', 'flake8_builtins', ''), - PipPackage('flake8-class-newline', 'flake8_class_newline', ''), - PipPackage('flake8-comprehensions', 'flake8_comprehensions', ''), - PipPackage('flake8-deprecated', 'flake8_deprecated', ''), - PipPackage('flake8-docstrings', 'flake8_docstrings', ''), - PipPackage('flake8-import-order', 'flake8_import_order', ''), - PipPackage('flake8-quotes', 'flake8_quotes', ''), - PipPackage('importlib-metadata', 'importlib_metadata', ''), - PipPackage('lark', 'lark', '==1.1.1'), - PipPackage('mypy', 'mypy', '==0.931'), - PipPackage('nose', 'nose', ''), - PipPackage('osrf-pycommon', 'osrf_pycommon', ''), - PipPackage('pathspec', 'pathspec', ''), - PipPackage('pydocstyle', 'pydocstyle', ''), - PipPackage('pyflakes', 'pyflakes', ''), - PipPackage('pyparsing', 'pyparsing', '==2.4.7'), - PipPackage('pytest', 'pytest', '==6.2.5'), - PipPackage('pytest-cov', 'pytest_cov', ''), - PipPackage('pytest-mock', 'pytest_mock', ''), - PipPackage('pytest-repeat', 'pytest_repeat', ''), - PipPackage('pytest-rerunfailures', 'pytest_rerunfailures', ''), - PipPackage('pytest-runner', 'ptr', ''), - PipPackage('pytest-timeout', 'pytest_timeout', '==2.1.0'), - PipPackage('pyyaml', 'yaml', ''), - PipPackage('setuptools', 'setuptools', '==59.6.0'), - PipPackage('vcstool', 'vcstool', ''), - PipPackage('yamllint', 'yamllint', ''), -] - colcon_packages = [ PipPackage('colcon-core', 'colcon_core', ''), PipPackage('colcon-defaults', 'colcon_defaults', ''), @@ -197,9 +161,6 @@ def __call__(self, parser, namespace, values, option_string=None): parser.add_argument( '--force-ansi-color', default=False, action='store_true', help="forces this program to output ansi color") - parser.add_argument( - '--ros-distro', required=True, - help="The ROS distribution being built") parser.add_argument( '--colcon-mixin-url', default=None, help='A mixin index url to be included by colcon') @@ -453,78 +414,10 @@ def run(args, build_function, blacklisted_package_names=None): # Now inside of the workspace... with change_directory(args.workspace): - def need_package_from_pipy(pkg_name): - try: - importlib.import_module(pkg_name) - except ModuleNotFoundError: - return True - - return False - - print('# BEGIN SUBSECTION: install Python packages') - # Print setuptools version - job.run(['"%s"' % job.python, '-c', '"import setuptools; print(setuptools.__version__)"'], - shell=True) - - # Print the pip version - job.run(['"%s"' % job.python, '-m', 'pip', '--version'], shell=True) - - # Install pip dependencies - pip_packages = [] - constraints = [] - - potential_pip_packages = pip_dependencies - if not args.colcon_branch: - potential_pip_packages += colcon_packages - - # We prefer to get packages from the distribution if they are already installed. - # If not, we add to the list to install from pip. - for pkgname, importname, version in pip_dependencies: - if need_package_from_pipy(importname): - pip_packages.append(pkgname) - # Even if we don't need to install the package, we still add the constraints - # (if they exist) so that other package installations will respect these. - if version: - constraints.append(f'{pkgname}{version}') - - if sys.platform == 'win32': - # Install fork of pyreadline containing fix for deprecation warnings - # TODO(jacobperron): Until upstream issue is resolved https://github.com/pyreadline/pyreadline/issues/65 - pip_packages += ['git+https://github.com/osrf/pyreadline'] - - # Setuptools > 61 somehow have broken Windows Debug. Pin it to 59.6.0 here which - # matches Ubuntu Jammy, and wait until upstream setuptools settles down. - pip_packages += ["setuptools==59.6.0"] - - pip_packages += ['cryptography', 'lxml', 'numpy'] - if args.ros_distro in ('humble', 'iron'): - pip_packages.append('netifaces') - - # to ensure that the build type specific package is installed - job.run( - ['"%s"' % job.python, '-m', 'pip', 'uninstall', '-y'] + - ['cryptography', 'lxml', 'numpy'], shell=True) - - if pip_packages: - print('Using constraints:') - print('\n'.join(constraints)) - with open('constraints.txt', 'w') as outfp: - outfp.write('\n'.join(constraints) + '\n') - - pip_cmd = ['"%s"' % job.python, '-m', 'pip', 'install', '-c', 'constraints.txt', '-U'] - if sys.platform == 'win32': - # Force reinstall so all dependencies are in virtual environment - # On Windows since we switch between the debug and non-debug - # interpreter all packages need to be reinstalled too - pip_cmd.append('--force-reinstall') - - job.run( - pip_cmd + pip_packages, - shell=True) - vcs_cmd = ['vcs'] if args.colcon_branch: + print('# BEGIN SUBSECTION: install custom colcon') # create .repos file for colcon repositories os.makedirs('colcon', exist_ok=True) with open('colcon/colcon.repos', 'w') as h: @@ -553,19 +446,30 @@ def need_package_from_pipy(pkg_name): ['"%s"' % job.python, '-m', 'pip', 'install', '-U'] + ['colcon/%s' % pkgname for pkgname, importname, version in colcon_packages], shell=True) + print('# END SUBSECTION') colcon_script = which('colcon') - # Show what pip has - job.run(['"%s"' % job.python, '-m', 'pip', 'freeze', '--all'], shell=True) - print('# END SUBSECTION') - # Fetch colcon mixins if args.colcon_mixin_url: + print('# BEGIN SUBSECTION: Fetch colcon mixins') true_cmd = 'VER>NUL' if sys.platform == 'win32' else 'true' job.run([colcon_script, 'mixin', 'remove', 'default', '||', true_cmd], shell=True) job.run([colcon_script, 'mixin', 'add', 'default', args.colcon_mixin_url], shell=True) job.run([colcon_script, 'mixin', 'update', 'default'], shell=True) + print('# END SUBSECTION') + + print('# BEGIN SUBSECTION: Print python versions') + # Print setuptools version + job.run(['"%s"' % job.python, '-c', '"import setuptools; print(setuptools.__version__)"'], + shell=True) + + # Print the pip version + job.run(['"%s"' % job.python, '-m', 'pip', '--version'], shell=True) + + # Show what pip has + job.run(['"%s"' % job.python, '-m', 'pip', 'list'], shell=True) + print('# END SUBSECTION') print('# BEGIN SUBSECTION: import repositories') repos_file_urls = [args.repo_file_url] diff --git a/ros2_batch_job/linux_batch/__init__.py b/ros2_batch_job/linux_batch/__init__.py index a922ef601..e206fa719 100644 --- a/ros2_batch_job/linux_batch/__init__.py +++ b/ros2_batch_job/linux_batch/__init__.py @@ -53,7 +53,7 @@ def show_env(self): # Show the env self.run(['export'], shell=True) # Show what pip has - self.run(['"%s"' % self.python, '-m', 'pip', 'freeze', '--all'], shell=True) + self.run(['"%s"' % self.python, '-m', 'pip', 'list'], shell=True) def setup_env(self): current_run = self.run diff --git a/ros2_batch_job/windows_batch/__init__.py b/ros2_batch_job/windows_batch/__init__.py index 25aacd3ee..a7aeb62a4 100644 --- a/ros2_batch_job/windows_batch/__init__.py +++ b/ros2_batch_job/windows_batch/__init__.py @@ -36,7 +36,7 @@ def show_env(self): # Show the env self.run(['set'], shell=True) # Show what pip has - self.run([self.python, '-m', 'pip', 'freeze', '--all']) + self.run([self.python, '-m', 'pip', 'list']) def setup_env(self): # Generate the env file diff --git a/windows_docker_resources/Dockerfile b/windows_docker_resources/Dockerfile index 4679e19e3..878919348 100644 --- a/windows_docker_resources/Dockerfile +++ b/windows_docker_resources/Dockerfile @@ -18,53 +18,50 @@ ARG WINDOWS_RELEASE_VERSION=$WINDOWS_RELEASE_ID # Use --isolation=process if you need to build in a mounted volume FROM mcr.microsoft.com/windows/server:$WINDOWS_RELEASE_VERSION -# Install cinc-solo, a compiled binary of chef-solo -RUN powershell "iex ((New-Object System.Net.WebClient).DownloadString('https://omnitruck.cinc.sh/install.ps1')); install -version 16.15.22" -# Update certificate bundle to work Let's Encrypt root certificate expiration -# https://www.openssl.org/blog/blog/2021/09/13/LetsEncryptRootCertExpire/ -# (in the parlance of the above post we're using workaround 1) -# This workaround is being incorporated directly in future releases of Cinc 16 and 17. -# Our application of the work around should be removed when updating to such a version. -COPY cacert.pem c:\cinc-project\cinc\embedded\ssl\certs\cacert.pem -COPY cacert.pem c:\cinc-project\cinc\embedded\lib\ruby\gems\2.7.0\gems\httpclient-2.8.3\lib\httpclient\cacert.pem +# Download and install all versions of MSVC we support. +# While this makes the images larger, it also means that we are certain to +# reuse the Docker layer cache, which should speed everything up. +# run_ros2_batch.py will source the appropriate one. -# Install Chocolatey by powershell script +# Install MSVC 2019 +RUN powershell -noexit irm https://aka.ms/vs/16/release/vs_buildtools.exe -OutFile vs_buildtools_2019.exe +RUN vs_buildtools_2019.exe --quiet --wait --norestart --add Microsoft.Component.MSBuild --add Microsoft.Net.Component.4.6.1.TargetingPack --add Microsoft.Net.Component.4.8.SDK --add Microsoft.VisualStudio.Component.CoreBuildTools --add Microsoft.VisualStudio.Component.Roslyn.Compiler --add Microsoft.VisualStudio.Component.TextTemplating --add Microsoft.VisualStudio.Component.VC.CLI.Support --add Microsoft.VisualStudio.Component.VC.CoreBuildTools --add Microsoft.VisualStudio.Component.VC.CoreIde --add Microsoft.VisualStudio.Component.VC.Redist.14.Latest --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 --add Microsoft.VisualStudio.Component.Windows10SDK --add Microsoft.VisualStudio.Component.Windows10SDK.19041 --add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core --add Microsoft.VisualStudio.Workload.MSBuildTools --add Microsoft.VisualStudio.Workload.VCTools -# Pinning chocolatey version to 1.4.0 previous one to major bump to 2.0.0 due to issues with chocolatey_package chef resource -# See https://github.com/chef/chef/issues/13751 for more detail of the issue -# This should be solved for chef version 18; see https://github.com/chef/chef/pull/13833 -RUN powershell -noexit "$env:chocolateyDownloadUrl = 'https://community.chocolatey.org/api/v2/package/chocolatey/1.4.0'; Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" +# Install MSVC 2022 +RUN powershell -noexit irm https://aka.ms/vs/17/release/vs_buildtools.exe -OutFile vs_buildtools_2022.exe +RUN vs_buildtools_2022.exe --quiet --wait --norestart --add Microsoft.Component.MSBuild --add Microsoft.Net.Component.4.6.1.TargetingPack --add Microsoft.Net.Component.4.8.SDK --add Microsoft.VisualStudio.Component.CoreBuildTools --add Microsoft.VisualStudio.Component.Roslyn.Compiler --add Microsoft.VisualStudio.Component.TextTemplating --add Microsoft.VisualStudio.Component.VC.CLI.Support --add Microsoft.VisualStudio.Component.VC.CoreBuildTools --add Microsoft.VisualStudio.Component.VC.CoreIde --add Microsoft.VisualStudio.Component.VC.Redist.14.Latest --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 --add Microsoft.VisualStudio.Component.Windows10SDK --add Microsoft.VisualStudio.Component.Windows10SDK.19041 --add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core --add Microsoft.VisualStudio.Workload.MSBuildTools --add Microsoft.VisualStudio.Workload.VCTools -# choco installs. chef-workstation is being installed to get berks and download cookbook dependencies -RUN choco install -y git chef-workstation +# Install pixi +RUN powershell -noexit "Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://pixi.sh/install.ps1'))" -# Copy over necessary files into container +# Install dependencies via pixi +ARG ROS_DISTRO=rolling +WORKDIR C:\pixi_ws +RUN powershell -noexit irm https://raw.githubusercontent.com/ros2/ros2/refs/heads/$ROS_DISTRO/pixi.toml -OutFile pixi.toml +RUN pixi --color never --no-progress -q install +RUN pixi --color never --no-progress -q list + +# Setup environment variables needed for Connext +ENV RTI_LICENSE_FILE C:\connext\rti_license.dat +ENV CONNEXTDDS_DIR C:\connext\rti_connext_dds-6.0.1 +ENV RTI_OPENSSL_BIN C:\connext\openssl-1.1.1k\x64Win64VS2017\bin +ENV RTI_OPENSSL_LIB C:\connext\openssl-1.1.1k\x64Win64VS2017\lib + +# Copy over necessary Connext files into container RUN IF NOT EXIST "C:\TEMP" mkdir C:\TEMP -COPY rticonnextdds-license\ C:\TEMP\rticonnextdds-license COPY rticonnextdds-src\ C:\TEMP\rticonnextdds-src RUN copy /b C:\TEMP\rticonnextdds-src\rti_connext_dds-6.0.1-pro-host-x64Win64.exe.??? C:\TEMP\rticonnextdds-src\rti_connext_dds-6.0.1-pro-host-x64Win64.exe RUN copy /b C:\TEMP\rticonnextdds-src\rti_connext_dds-6.0.1-pro-target-x64Win64VS2017.rtipkg.??? C:\TEMP\rticonnextdds-src\rti_connext_dds-6.0.1-pro-target-x64Win64VS2017.rtipkg +COPY rticonnextdds-license\rti_license.dat $RTI_LICENSE_FILE -# ROS_DISTRO argument should be set to install dependencies for the target ROS version. -ARG ROS_DISTRO - -COPY install_ros2_${ROS_DISTRO}.json C:\TEMP\ -COPY solo.rb C:\TEMP\ -COPY ros2-cookbooks\ C:\TEMP\ros2-cookbooks -RUN IF NOT EXIST "C:\TEMP" mkdir C:\TEMP\environments -COPY qtaccount\ros2ci.rb C:\TEMP\environments\ros2ci.rb - -# Download vendor cookbooks -WORKDIR C:\TEMP\ros2-cookbooks\cookbooks\ros2_windows -RUN C:\opscode\chef-workstation\bin\berks vendor C:\TEMP\ros2-cookbooks\cookbooks +# Install Connext +RUN pixi run 7z x -oC:\connext C:\TEMP\rticonnextdds-src\openssl-1.1.1k-target-x64Win64VS2017.zip +RUN C:\TEMP\rticonnextdds-src\rti_connext_dds-6.0.1-pro-host-x64Win64.exe --mode unattended --unattendedmodeui minimalWithDialogs --prefix C:\connext +RUN %CONNEXTDDS_DIR%\bin\rtipkginstall.bat -u C:\TEMP\rticonnextdds-src\openssl-1.1.1k-6.0.1.25-host-x64Win64.rtipkg +RUN %CONNEXTDDS_DIR%\bin\rtipkginstall.bat -u C:\TEMP\rticonnextdds-src\rti_connext_dds-6.0.1-pro-target-x64Win64VS2017.rtipkg +RUN %CONNEXTDDS_DIR%\bin\rtipkginstall.bat -u C:\TEMP\rticonnextdds-src\rti_security_plugins-6.0.1.25-host-x64Win64.rtipkg +RUN %CONNEXTDDS_DIR%\bin\rtipkginstall.bat -u C:\TEMP\rticonnextdds-src\rti_security_plugins-6.0.1.25-target-x64Win64VS2017.rtipkg -# Initial run -RUN c:\cinc-project\cinc\bin\cinc-solo.bat -c C:\TEMP\solo.rb -Eros2ci -j C:\TEMP\install_ros2_%ROS_DISTRO%.json - -# Invalidate daily to run updates -RUN echo "@todays_date" -RUN c:\cinc-project\cinc\bin\cinc-solo.bat -c C:\TEMP\solo.rb -Eros2ci -j C:\TEMP\install_ros2_%ROS_DISTRO%.json - -WORKDIR C:\ci # Note that this *must* be in shell form, not exec form, so Docker on Windows appropriately substitutes %CI_ARGS% -CMD "python run_ros2_batch.py %CI_ARGS%" +WORKDIR C:\ci +CMD "pixi run --manifest-path C:\pixi_ws\pixi.toml python run_ros2_batch.py %CI_ARGS%" diff --git a/windows_docker_resources/README.md b/windows_docker_resources/README.md index b2e12d839..325eecbb7 100644 --- a/windows_docker_resources/README.md +++ b/windows_docker_resources/README.md @@ -1,69 +1,8 @@ -## Contributing changes to Windows ROS 2 Chef Cookbook +# Contributing changes to Windows CI -Running the Windows CI agent as a configurable chef cookbook follows the general benefits of "infrastructure as software". -Making changes can be submitted as pull requests, which in turn can be reviewed and tested before they are deployed. -Below is the recommended process for submitting changes to the ROS 2 chef cookbook for Windows for use with ci.ros2.org. +ROS 2 Windows CI uses [pixi](https://pixi.sh/latest/) to manage dependencies in a pixi.toml file, one per ROS distribution. +These pixi.toml files are stored in a centralized repository at https://github.com/ros2/ros2 on the branches that correspond to the ROS distribution. +They are stored separately from this repository so they can be referenced both by this CI, as well as the installation instructions. -### Step 1. Create a ros2-cookbooks PR -Open a PR at github.com/ros-infrastructure/ros2-cookbooks for your desired change. - -### Step 2. Create a ros2/ci PR -Open corresponding PR at github.com/ros2/ci which updates the ros2-cookbooks git submodule to your new branch of the ros2-cookbooks repo. - -### Step 3. Verify the ros2-cookbooks PR -It can take a while to test changes to the cookbook directly on CI, so it is strongly advised to test locally and verify your change inside a representative docker container. -See the following section for information on local testing. -After you have tested it locally, submit jobs to https://ci.ros2.org/job/ci_windows and set CI_SCRIPTS_BRANCH to your ros2/ci PR branch. -You may need to verify the PR for several different ROS 2 releases. -You can set `CI_ROS_DISTRO` on ci.ros2.org, which will choose the corresponding chef cookbook configuration to test your updates. -If testing on different releases, don't forget to specify the correct ros2.repos file. - -#### Testing locally -Do the following on your own machine or VM. - -Get the Windows release version with the following powershell command -``` -powershell $(Get-ItemProperty -Path 'HKLM:SOFTWARE\Microsoft\Windows NT\CurrentVersion\Update\TargetingInfo\Installed\Server.OS.amd64' -Name Version).Version -``` - -Change your directory to ci repo directory and run the following docker build command. -Replace the value of WINDOWS_RELEASE_VERSION with the string you found above. -``` -docker build --build-arg WINDOWS_RELEASE_VERSION=10.0.18363.900 -t ros2_windows_ci windows_docker_resources -``` - -If it builds correctly, you can be assured your changes to the chef cookbook has been parsed and used by chef correctly. -However, you should also check that your change doesn't affect the build of ros2. -To do that, you need to run the docker container with representative arguments. - -You will need to run the following from a developer command prompt with administrator privileges. -Set a variable with the appropriate CI_ARGS, to test up to rclcpp. - -``` -set CI_ARGS=--force-ansi-color --workspace-path C:\J\workspace\ci_windows --ros-distro foxy --ignore-rmw rmw_fastrtps_dynamic_cpp --colcon-mixin-url https://raw.githubusercontent.com/colcon/colcon-mixin-repository/master/index.yaml --visual-studio-version 2019 --build-args --event-handlers console_cohesion+ console_package_list+ --cmake-args -DINSTALL_EXAMPLES=OFF -DSECURITY=ON --packages-up-to rclcpp --test-args --event-handlers console_direct+ --executor sequential --retest-until-pass 2 --ctest-args -LE xfail --pytest-args -m \"not xfail\" --packages-up-to rclcpp``` -``` - -Run the docker container with these arguments -``` -docker run --isolation=process -e ROS_DOMAIN_ID=1 -e CI_ARGS=%CI_ARGS% -v "C:\J\workspace\ci_windows":"C:\ci" ros2_windows_ci -``` - -rclcpp may not be the correct package to test for your change. -Choose a package to test up to that adequately ensures your change works as intended. - -### Step 4. Merge the ros2-cookbooks PR -After your ros2-cookbooks PR is approved and passes testing, merge this PR first. -Because the git submodule has not yet been updated in ros2/ci, this merge will not change the behavior of ci.ros2.org. - -### Step 5. Update the ros2/ci PR -You will need to update your ros2/ci PR to use the master branch of ros2-cookbooks. -Update the submodule with the latest commit from your PR and target the main branch. - -### Step 6. Rerun the ci job for the ros2/ci PR -This will help verify that that the updated git submodule points to the correct commit. - -### Step 7. Merge the ros2/ci PR -Once you merge the PR, the docker containers will need to be rebuilt on the ci_windows nodes. -This can take some time and will slow down the first build for each node. -Choose a time of day when activity is low, so your change doesn't negatively impact other contributors. -This author uses 5-6PM US Pacific (1am GMT), which gives some time to revert your change before the nigthly builds should it have introduced an issue. +Thus, in order to update dependencies for this ROS 2 CI, the only required step is to open a PR to the appropriate pixi.toml file on https://github.com/ros2/ros2 . +Once that PR has been approved and merged, subsequent builds on ROS 2 CI will automatically fetch that dependency file. diff --git a/windows_docker_resources/install_ros2_humble.json b/windows_docker_resources/install_ros2_humble.json deleted file mode 100644 index d86e20f60..000000000 --- a/windows_docker_resources/install_ros2_humble.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "ros2_windows": { - "download_sources": "false", - "vs_version": "buildtools", - "vs_release": "@vs_version", - "qt5": { - "installation_method": "offline" - }, - "ros2_ws": "C:/ci", - "ros_distro": "humble", - "rti_connext": { - "target_platform": "x64Win64", - "min_vs_version": "2017", - "license_file": "C:/TEMP/rticonnextdds-license/rti_license.dat", - "installer_dir": "C:/TEMP/rticonnextdds-src", - "version": "6.0.1", - "edition": "pro", - "openssl_version": "1.1.1k" - } - }, - "run_list": [ - "recipe[ros2_windows::ros2]", - "recipe[ros2_windows::rti_connext]" - ] -} diff --git a/windows_docker_resources/install_ros2_iron.json b/windows_docker_resources/install_ros2_iron.json deleted file mode 100644 index bd2d4af8c..000000000 --- a/windows_docker_resources/install_ros2_iron.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "ros2_windows": { - "download_sources": "false", - "vs_version": "buildtools", - "vs_release": "@vs_version", - "qt5": { - "installation_method": "offline" - }, - "ros2_ws": "C:/ci", - "ros_distro": "iron", - "rti_connext": { - "target_platform": "x64Win64", - "min_vs_version": "2017", - "license_file": "C:/TEMP/rticonnextdds-license/rti_license.dat", - "installer_dir": "C:/TEMP/rticonnextdds-src", - "version": "6.0.1", - "edition": "pro", - "openssl_version": "1.1.1k" - } - }, - "run_list": [ - "recipe[ros2_windows::ros2]", - "recipe[ros2_windows::rti_connext]" - ] -} diff --git a/windows_docker_resources/install_ros2_jazzy.json b/windows_docker_resources/install_ros2_jazzy.json deleted file mode 100644 index 015b63026..000000000 --- a/windows_docker_resources/install_ros2_jazzy.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "ros2_windows": { - "download_sources": "false", - "vs_version": "buildtools", - "vs_release": "@vs_version", - "qt5": { - "installation_method": "offline" - }, - "ros2_ws": "C:/ci", - "ros_distro": "jazzy", - "rti_connext": { - "target_platform": "x64Win64", - "min_vs_version": "2017", - "license_file": "C:/TEMP/rticonnextdds-license/rti_license.dat", - "installer_dir": "C:/TEMP/rticonnextdds-src", - "version": "6.0.1", - "edition": "pro", - "openssl_version": "1.1.1k" - } - }, - "run_list": [ - "recipe[ros2_windows::ros2]", - "recipe[ros2_windows::rti_connext]" - ] -} diff --git a/windows_docker_resources/install_ros2_rolling.json b/windows_docker_resources/install_ros2_rolling.json deleted file mode 100644 index 5e2708d90..000000000 --- a/windows_docker_resources/install_ros2_rolling.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "ros2_windows": { - "download_sources": "false", - "vs_version": "buildtools", - "vs_release": "@vs_version", - "qt5": { - "installation_method": "offline" - }, - "ros2_ws": "C:/ci", - "ros_distro": "rolling", - "rti_connext": { - "target_platform": "x64Win64", - "min_vs_version": "2017", - "license_file": "C:/TEMP/rticonnextdds-license/rti_license.dat", - "installer_dir": "C:/TEMP/rticonnextdds-src", - "version": "6.0.1", - "edition": "pro", - "openssl_version": "1.1.1k" - } - }, - "run_list": [ - "recipe[ros2_windows::ros2]", - "recipe[ros2_windows::rti_connext]" - ] -} diff --git a/windows_docker_resources/qtaccount b/windows_docker_resources/qtaccount deleted file mode 160000 index db1853133..000000000 --- a/windows_docker_resources/qtaccount +++ /dev/null @@ -1 +0,0 @@ -Subproject commit db1853133789fd171a91b4170cc3838e81dacc0f diff --git a/windows_docker_resources/ros2-cookbooks b/windows_docker_resources/ros2-cookbooks deleted file mode 160000 index 96cbbebd4..000000000 --- a/windows_docker_resources/ros2-cookbooks +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 96cbbebd449800d20e3db58716a3c2271c55c18d diff --git a/windows_docker_resources/solo.rb b/windows_docker_resources/solo.rb deleted file mode 100644 index bb20ea8f3..000000000 --- a/windows_docker_resources/solo.rb +++ /dev/null @@ -1,3 +0,0 @@ -cookbook_path [File.join(File.expand_path("..", __FILE__), "ros2-cookbooks", "cookbooks")] -environment_path File.join(File.expand_path("..", __FILE__), "environments") -solo true From b7b5b520203deb9efa7c5a15ee39696f9d970a25 Mon Sep 17 00:00:00 2001 From: Chris Lalancette Date: Wed, 5 Feb 2025 10:48:49 -0500 Subject: [PATCH 2/5] Use Windows-style environment variable. Signed-off-by: Chris Lalancette --- windows_docker_resources/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows_docker_resources/Dockerfile b/windows_docker_resources/Dockerfile index 878919348..cf48f6db5 100644 --- a/windows_docker_resources/Dockerfile +++ b/windows_docker_resources/Dockerfile @@ -37,7 +37,7 @@ RUN powershell -noexit "Set-ExecutionPolicy Bypass -Scope Process -Force; iex (( # Install dependencies via pixi ARG ROS_DISTRO=rolling WORKDIR C:\pixi_ws -RUN powershell -noexit irm https://raw.githubusercontent.com/ros2/ros2/refs/heads/$ROS_DISTRO/pixi.toml -OutFile pixi.toml +RUN powershell -noexit irm https://raw.githubusercontent.com/ros2/ros2/refs/heads/%ROS_DISTRO%/pixi.toml -OutFile pixi.toml RUN pixi --color never --no-progress -q install RUN pixi --color never --no-progress -q list From 381c5842a5a64f7fe731380e8bdf83464a3e5fc1 Mon Sep 17 00:00:00 2001 From: Chris Lalancette Date: Thu, 6 Feb 2025 15:43:05 -0500 Subject: [PATCH 3/5] Remove unnecessary shell=True. Signed-off-by: Chris Lalancette --- ros2_batch_job/__main__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ros2_batch_job/__main__.py b/ros2_batch_job/__main__.py index b026b89f3..0580c56a0 100644 --- a/ros2_batch_job/__main__.py +++ b/ros2_batch_job/__main__.py @@ -461,8 +461,7 @@ def run(args, build_function, blacklisted_package_names=None): print('# BEGIN SUBSECTION: Print python versions') # Print setuptools version - job.run(['"%s"' % job.python, '-c', '"import setuptools; print(setuptools.__version__)"'], - shell=True) + job.run(['"%s"' % job.python, '-c', '"import setuptools; print(setuptools.__version__)"']) # Print the pip version job.run(['"%s"' % job.python, '-m', 'pip', '--version'], shell=True) From dd9ffb1419e1e83c9bcbb89588b96b929f9d1ad5 Mon Sep 17 00:00:00 2001 From: Chris Lalancette Date: Fri, 7 Feb 2025 13:06:42 -0500 Subject: [PATCH 4/5] Revert "Remove unnecessary shell=True." This reverts commit 381c5842a5a64f7fe731380e8bdf83464a3e5fc1. --- ros2_batch_job/__main__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ros2_batch_job/__main__.py b/ros2_batch_job/__main__.py index 0580c56a0..b026b89f3 100644 --- a/ros2_batch_job/__main__.py +++ b/ros2_batch_job/__main__.py @@ -461,7 +461,8 @@ def run(args, build_function, blacklisted_package_names=None): print('# BEGIN SUBSECTION: Print python versions') # Print setuptools version - job.run(['"%s"' % job.python, '-c', '"import setuptools; print(setuptools.__version__)"']) + job.run(['"%s"' % job.python, '-c', '"import setuptools; print(setuptools.__version__)"'], + shell=True) # Print the pip version job.run(['"%s"' % job.python, '-m', 'pip', '--version'], shell=True) From b5537623ce010ff7b00cc3c8cca2b6f802afc568 Mon Sep 17 00:00:00 2001 From: Chris Lalancette Date: Fri, 7 Feb 2025 13:16:38 -0500 Subject: [PATCH 5/5] Restore the testing locally instructions. Signed-off-by: Chris Lalancette --- windows_docker_resources/README.md | 32 ++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/windows_docker_resources/README.md b/windows_docker_resources/README.md index 325eecbb7..63f0a28ee 100644 --- a/windows_docker_resources/README.md +++ b/windows_docker_resources/README.md @@ -6,3 +6,35 @@ They are stored separately from this repository so they can be referenced both b Thus, in order to update dependencies for this ROS 2 CI, the only required step is to open a PR to the appropriate pixi.toml file on https://github.com/ros2/ros2 . Once that PR has been approved and merged, subsequent builds on ROS 2 CI will automatically fetch that dependency file. + +## Testing locally + +Do the following on your own machine or VM. + +Get the Windows release version with the following powershell command +``` +powershell $(Get-ItemProperty -Path 'HKLM:SOFTWARE\Microsoft\Windows NT\CurrentVersion\Update\TargetingInfo\Installed\Server.OS.amd64' -Name Version).Version +``` + +Change your directory to ci repo directory and run the following docker build command. +Replace the value of WINDOWS_RELEASE_VERSION with the string you found above. +``` +docker build --build-arg WINDOWS_RELEASE_VERSION=10.0.18363.900 --build-arg ROS_DISTRO=rolling -t ros2_windows_ci -f windows_docker_resources\Dockerfile windows_docker_resources +``` + +To actually run a build, you need to run the docker container with with representative arguments. + +You will need to run the following from a developer command prompt with administrator privileges. +Set a variable with the appropriate CI_ARGS, to test up to rclcpp. + +``` +set CI_ARGS=--force-ansi-color --workspace-path C:\J\workspace\ci_windows --ignore-rmw rmw_fastrtps_dynamic_cpp --repo-file-url https://raw.githubusercontent.com/ros2/ros2/rolling/ros2.repos --colcon-mixin-url https://raw.githubusercontent.com/colcon/colcon-mixin-repository/master/index.yaml --visual-studio-version 2019 --build-args --event-handlers console_cohesion+ console_package_list+ --cmake-args -DINSTALL_EXAMPLES=OFF -DSECURITY=ON -DAPPEND_PROJECT_NAME_TO_INCLUDEDIR=ON --packages-up-to rclcpp --test-args --event-handlers console_direct+ --executor sequential --retest-until-pass 2 --ctest-args -LE xfail --pytest-args -m \"not xfail\" --packages-up-to rclcpp``` +``` + +Run the docker container with these arguments +``` +docker run --isolation=process --rm -e ROS_DOMAIN_ID=1 -e CI_ARGS="%CI_ARGS%" -v "C:\J\workspace\ci_windows":"C:\ci" ros2_windows_ci +``` + +rclcpp may not be the correct package to test for your change. +Choose a package to test up to that adequately ensures your change works as intended.