Skip to content

Commit

Permalink
Add a new build flag to build.py for using with vcpkg (#23723)
Browse files Browse the repository at this point in the history
1. **Add new flag to build.py**: Introduced a
`--use_vcpkg_ms_internal_asset_cache` flag to `build.py`. The flag is
intended for internal use only.
2. **Reduce excessive logs**: Removed some excessive logs from
`vcpkg_helper.py`.
  • Loading branch information
snnn authored Feb 20, 2025
1 parent b230c7b commit 3df43a2
Show file tree
Hide file tree
Showing 37 changed files with 82 additions and 82 deletions.
37 changes: 18 additions & 19 deletions tools/ci_build/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -494,6 +494,11 @@ def convert_arg_line_to_args(self, arg_line):
action="store_true",
help="Use vcpkg to search dependencies. Requires CMAKE_TOOLCHAIN_FILE for vcpkg.cmake",
)
parser.add_argument(
"--use_vcpkg_ms_internal_asset_cache",
action="store_true",
help="Microsoft internal option",
)

# WebAssembly build
parser.add_argument("--build_wasm", action="store_true", help="Build for WebAssembly")
Expand Down Expand Up @@ -1064,26 +1069,20 @@ def generate_vcpkg_install_options(build_dir, args):
temp_dir = os.environ["AGENT_TEMPDIRECTORY"]
vcpkg_install_options.append(f"--x-buildtrees-root={temp_dir}")

SYSTEM_COLLECTIONURI = os.getenv("SYSTEM_COLLECTIONURI") # noqa: N806

# Config asset cache
terrapin_cmd_path = shutil.which("TerrapinRetrievalTool")
if terrapin_cmd_path is None:
terrapin_cmd_path = "C:\\local\\Terrapin\\TerrapinRetrievalTool.exe"
if not os.path.exists(terrapin_cmd_path):
terrapin_cmd_path = None
if terrapin_cmd_path is not None:
vcpkg_install_options.append(
"--x-asset-sources=x-script,"
+ terrapin_cmd_path
+ " -b https://vcpkg.storage.devpackages.microsoft.io/artifacts/ -a true -u Environment -p {url} -s {sha512} -d {dst}\\;x-block-origin"
)
else:
if (
SYSTEM_COLLECTIONURI == "https://dev.azure.com/onnxruntime/"
or SYSTEM_COLLECTIONURI == "https://dev.azure.com/aiinfra/"
or SYSTEM_COLLECTIONURI == "https://aiinfra.visualstudio.com/"
):
if args.use_vcpkg_ms_internal_asset_cache:
terrapin_cmd_path = shutil.which("TerrapinRetrievalTool")
if terrapin_cmd_path is None:
terrapin_cmd_path = "C:\\local\\Terrapin\\TerrapinRetrievalTool.exe"
if not os.path.exists(terrapin_cmd_path):
terrapin_cmd_path = None
if terrapin_cmd_path is not None:
vcpkg_install_options.append(
"--x-asset-sources=x-script,"
+ terrapin_cmd_path
+ " -b https://vcpkg.storage.devpackages.microsoft.io/artifacts/ -a true -u Environment -p {url} -s {sha512} -d {dst}\\;x-block-origin"
)
else:
vcpkg_install_options.append(
"--x-asset-sources=x-azurl,https://vcpkg.storage.devpackages.microsoft.io/artifacts/\\;x-block-origin"
)
Expand Down
4 changes: 3 additions & 1 deletion tools/ci_build/github/android/build_aar_package.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,9 @@ def _build_aar(args):
jnilibs_dir = os.path.join(intermediates_dir, "jnilibs", build_config)
exe_dir = os.path.join(intermediates_dir, "executables", build_config)
base_build_command = (
[sys.executable, BUILD_PY] + build_settings["build_params"] + ["--config=" + build_config, "--use_vcpkg"]
[sys.executable, BUILD_PY]
+ build_settings["build_params"]
+ ["--config=" + build_config, "--use_vcpkg", "--use_vcpkg_ms_internal_asset_cache"]
)
header_files_path = ""

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ jobs:
--android_ndk_path $ANDROID_NDK_HOME \
--android_abi=x86_64 \
--android_api=31 \
--parallel --use_vcpkg \
--parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache \
--build_shared_lib \
--use_qnn static_lib \
--qnn_home $(QnnSDKRootDir) \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ stages:
--android_abi=x86_64 \
--android_api=30 \
--skip_submodule_sync \
--parallel --use_vcpkg \
--parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache \
--cmake_generator=Ninja \
--build_java
displayName: CPU EP, Build and Test
Expand Down Expand Up @@ -146,7 +146,7 @@ stages:
--android_abi=x86_64 \
--android_api=29 \
--skip_submodule_sync \
--parallel --use_vcpkg \
--parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache \
--use_nnapi \
--build_shared_lib \
--cmake_generator=Ninja \
Expand Down Expand Up @@ -203,7 +203,7 @@ stages:
--android_abi=x86_64 \
--android_api=29 \
--skip_submodule_sync \
--parallel --use_vcpkg \
--parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache \
--use_nnapi \
--build_shared_lib \
--cmake_generator=Ninja \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ stages:
--config Release --update --build \
--skip_submodule_sync \
--build_shared_lib \
--parallel --use_vcpkg \
--parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache \
--build_wheel \
--enable_onnx_tests --use_cuda --cuda_version=11.8 --cuda_home=/usr/local/cuda-11.8 --cudnn_home=/usr/local/cuda-11.8 \
--enable_cuda_profiling \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ stages:
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
ArtifactName: 'drop-nuget-dml'
StageName: 'Windows_CI_GPU_DML_Dev'
BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --build_nodejs --cmake_generator "Visual Studio 17 2022" --use_vcpkg
BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --build_nodejs --cmake_generator "Visual Studio 17 2022" --use_vcpkg --use_vcpkg_ms_internal_asset_cache
BuildArch: 'x64'
msbuildArchitecture: 'amd64'
EnvSetupScript: 'setup_env.bat'
Expand All @@ -157,7 +157,7 @@ stages:
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
ArtifactName: 'drop-win-dml-x86-zip'
StageName: 'Windows_CI_GPU_DML_Dev_x86'
BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --cmake_generator "Visual Studio 17 2022" --use_vcpkg
BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --cmake_generator "Visual Studio 17 2022" --use_vcpkg --use_vcpkg_ms_internal_asset_cache
BuildArch: 'x86'
EnvSetupScript: 'setup_env_x86.bat'
sln_platform: 'Win32'
Expand All @@ -179,7 +179,7 @@ stages:
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
ArtifactName: 'drop-win-dml-arm64-zip'
StageName: 'Windows_CI_GPU_DML_Dev_arm64'
BuildCommand: --build_dir $(Build.BinariesDirectory) --arm64 --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --build_nodejs --cmake_generator "Visual Studio 17 2022" --use_vcpkg
BuildCommand: --build_dir $(Build.BinariesDirectory) --arm64 --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --build_nodejs --cmake_generator "Visual Studio 17 2022" --use_vcpkg --use_vcpkg_ms_internal_asset_cache
BuildArch: 'x64'
EnvSetupScript: 'setup_env.bat'
sln_platform: 'arm64'
Expand Down
6 changes: 3 additions & 3 deletions tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,15 +83,15 @@ stages:
--config Debug \
--skip_submodule_sync \
--build_shared_lib \
--parallel --use_vcpkg \
--parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache \
--enable_onnx_tests --enable_address_sanitizer \
--update --build;
python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Debug \
--skip_submodule_sync \
--build_shared_lib \
--parallel --use_vcpkg \
--parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache \
--enable_onnx_tests --enable_address_sanitizer \
--test;
'
Expand Down Expand Up @@ -199,7 +199,7 @@ stages:
--config Release \
--skip_submodule_sync \
--build_shared_lib \
--parallel --use_vcpkg --use_binskim_compliant_compile_flags \
--parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --use_binskim_compliant_compile_flags \
--build_wheel \
--build_csharp \
--enable_onnx_tests \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ jobs:
--config Debug Release \
--skip_submodule_sync \
--build_shared_lib \
--parallel --use_vcpkg \
--parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache \
--enable_pybind \
--enable_onnx_tests \
--build_java \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ stages:
- script: |
set -e -x
mkdir -p $HOME/.onnx
docker run -e SYSTEM_COLLECTIONURI --rm \
docker run --rm \
--volume /data/onnx:/data/onnx:ro \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \
Expand Down Expand Up @@ -163,7 +163,7 @@ stages:
script: |
set -e -x
mkdir -p $HOME/.onnx
docker run -e SYSTEM_COLLECTIONURI --gpus all --rm \
docker run --gpus all --rm \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory)/Release:/build/Release \
--volume /data/models:/build/models:ro \
Expand Down Expand Up @@ -191,7 +191,7 @@ stages:
script: |
set -e -x
mkdir -p $HOME/.onnx
docker run -e SYSTEM_COLLECTIONURI --gpus all --shm-size=1g --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 --rm \
docker run --gpus all --shm-size=1g --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 --rm \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory)/Release:/build/Release \
--volume /data/models:/build/models:ro \
Expand All @@ -209,7 +209,7 @@ stages:
cd /onnxruntime_src/java && /onnxruntime_src/java/gradlew cmakeCheck -DcmakeBuildDir=/build/Release -DUSE_CUDA=1; \
cd /tmp; \
python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --config Release --test --skip_submodule_sync --build_shared_lib --parallel --use_vcpkg --use_binskim_compliant_compile_flags --build_wheel --enable_onnx_tests \
--build_dir /build --config Release --test --skip_submodule_sync --build_shared_lib --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --use_binskim_compliant_compile_flags --build_wheel --enable_onnx_tests \
--enable_transformers_tool_test --use_cuda --cuda_version=${{parameters.CudaVersion}} --cuda_home=/usr/local/cuda --cudnn_home=/usr/local/cuda \
--enable_pybind --build_java --ctest_path "" ; \
'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ jobs:
--update \
--build_dir /build \
--build \
--parallel --use_vcpkg \
--parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache \
--build_wheel \
--skip_submodule_sync \
--use_cache \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ jobs:
--use_qnn $(QnnLibKind) \
--qnn_home $(QnnSDKRootDir) \
--cmake_generator=Ninja \
--update --build --parallel --use_vcpkg
--update --build --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache
displayName: Build QNN EP
- script: |
Expand All @@ -82,7 +82,7 @@ jobs:
--use_qnn $(QnnLibKind) \
--qnn_home $(QnnSDKRootDir) \
--cmake_generator=Ninja \
--test --use_vcpkg
--test --use_vcpkg --use_vcpkg_ms_internal_asset_cache
displayName: Run unit tests
- task: CmdLine@2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ jobs:
--enable_nccl \
--build_dir /build \
--build_shared_lib \
--parallel --use_vcpkg \
--parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache \
--build_wheel \
--skip_submodule_sync \
--test --enable_onnx_tests --enable_transformers_tool_test \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ jobs:
--build_dir build \
--skip_submodule_sync \
--cmake_generator=Ninja \
--parallel --use_vcpkg --use_binskim_compliant_compile_flags \
--parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --use_binskim_compliant_compile_flags \
--build_shared_lib \
--config Debug \
--use_cache \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ stages:
- ${{ else }}:
- powershell: |
python tools\ci_build\build.py ${{ parameters.BuildCommand }} --use_binskim_compliant_compile_flags --parallel --use_vcpkg --build_csharp --build --update --config $(BuildConfig) --build_nuget --msbuild_extra_options IncludeMobileTargets=false ${{ variables.build_py_lto_flag }}
python tools\ci_build\build.py ${{ parameters.BuildCommand }} --use_binskim_compliant_compile_flags --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --build_csharp --build --update --config $(BuildConfig) --build_nuget --msbuild_extra_options IncludeMobileTargets=false ${{ variables.build_py_lto_flag }}
- ${{ if notIn(parameters['sln_platform'], 'Win32', 'x64') }}:
# Use cross-compiled protoc
Expand Down
12 changes: 6 additions & 6 deletions tools/ci_build/github/azure-pipelines/post-merge-jobs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ stages:
buildArch: x64
additionalBuildFlags: --build_wheel --build_java --build_nodejs --use_cuda --cuda_home="$(Agent.TempDirectory)\v${{ parameters.CudaVersion }}"
--enable_cuda_profiling --enable_transformers_tool_test
--use_vcpkg --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
--use_vcpkg --use_vcpkg_ms_internal_asset_cache --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
--cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=ON
--cmake_extra_defines onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS=ON
msbuildPlatform: x64
Expand All @@ -247,7 +247,7 @@ stages:
BuildConfig: 'RelWithDebInfo'
EnvSetupScript: setup_env_trt.bat
buildArch: x64
additionalBuildFlags: --config RelWithDebInfo --parallel --use_binskim_compliant_compile_flags --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\${{ variables.win_trt_folder }}" --cuda_home="$(Agent.TempDirectory)\v${{ parameters.CudaVersion }}" --use_vcpkg --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
additionalBuildFlags: --config RelWithDebInfo --parallel --use_binskim_compliant_compile_flags --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\${{ variables.win_trt_folder }}" --cuda_home="$(Agent.TempDirectory)\v${{ parameters.CudaVersion }}" --use_vcpkg --use_vcpkg_ms_internal_asset_cache --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
msbuildPlatform: x64
isX86: false
job_name_suffix: x64_RelWithDebInfo
Expand All @@ -265,7 +265,7 @@ stages:
BuildConfig: 'Debug'
EnvSetupScript: setup_env.bat
buildArch: x64
additionalBuildFlags: --disable_memleak_checker --use_mimalloc --use_vcpkg
additionalBuildFlags: --disable_memleak_checker --use_mimalloc --use_vcpkg --use_vcpkg_ms_internal_asset_cache
msbuildPlatform: x64
isX86: false
job_name_suffix: x64_mimalloc
Expand All @@ -284,7 +284,7 @@ stages:
BuildConfig: 'Debug'
EnvSetupScript: setup_env.bat
buildArch: x64
additionalBuildFlags: --use_vcpkg --cmake_extra_defines onnxruntime_ENABLE_MEMORY_PROFILE=ON
additionalBuildFlags: --use_vcpkg --use_vcpkg_ms_internal_asset_cache --cmake_extra_defines onnxruntime_ENABLE_MEMORY_PROFILE=ON
msbuildPlatform: x64
isX86: false
job_name_suffix: x64_no_memory_profiling
Expand Down Expand Up @@ -322,7 +322,7 @@ stages:
BuildConfig: 'Debug'
EnvSetupScript: setup_env.bat
buildArch: x64
additionalBuildFlags: --use_vcpkg --build_shared_lib --cmake_extra_defines onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS=ON
additionalBuildFlags: --use_vcpkg --use_vcpkg_ms_internal_asset_cache --build_shared_lib --cmake_extra_defines onnxruntime_DEBUG_NODE_INPUTS_OUTPUTS=ON
msbuildPlatform: x64
isX86: false
job_name_suffix: x64_debug_node_input_output
Expand Down Expand Up @@ -353,7 +353,7 @@ stages:
set -e
ln -s /data/models .
#Build onnxruntime and run the instrumented program(unitests)
LLVM_PROFILE_FILE="%p.profraw" CFLAGS="-g -fprofile-instr-generate -fcoverage-mapping" CXXFLAGS="-g -fprofile-instr-generate -fcoverage-mapping" CC=clang CXX=clang++ python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir=$(Build.BinariesDirectory) --config Debug --parallel --use_vcpkg --skip_submodule_sync --build_shared_lib --enable_onnx_tests --cmake_extra_defines RUN_MODELTEST_IN_DEBUG_MODE=ON
LLVM_PROFILE_FILE="%p.profraw" CFLAGS="-g -fprofile-instr-generate -fcoverage-mapping" CXXFLAGS="-g -fprofile-instr-generate -fcoverage-mapping" CC=clang CXX=clang++ python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir=$(Build.BinariesDirectory) --config Debug --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --skip_submodule_sync --build_shared_lib --enable_onnx_tests --cmake_extra_defines RUN_MODELTEST_IN_DEBUG_MODE=ON
cd Debug
./onnxruntime_mlas_test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ stages:
set -e -x
export _PYTHON_HOST_PLATFORM=macosx-${{variables.MACOSX_DEPLOYMENT_TARGET}}-universal2
python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt'
python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --use_coreml --skip_submodule_sync --parallel --use_vcpkg --use_binskim_compliant_compile_flags --config Release --build_wheel ${{ parameters.build_py_parameters }} --use_coreml --cmake_extra_defines CMAKE_OSX_ARCHITECTURES="arm64;x86_64" --update --build
python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --use_coreml --skip_submodule_sync --parallel --use_vcpkg --use_vcpkg_ms_internal_asset_cache --use_binskim_compliant_compile_flags --config Release --build_wheel ${{ parameters.build_py_parameters }} --use_coreml --cmake_extra_defines CMAKE_OSX_ARCHITECTURES="arm64;x86_64" --update --build
displayName: 'Command Line Script'
- script: |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ stages:
--cmake_generator "$(VSGenerator)"
--enable_pybind
--enable_onnx_tests
--parallel 8 --use_vcpkg --use_binskim_compliant_compile_flags --update --build --msvc_toolset 14.40
--parallel 8 --use_vcpkg --use_vcpkg_ms_internal_asset_cache --use_binskim_compliant_compile_flags --update --build --msvc_toolset 14.40
$(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }} ${{ variables.trt_build_flag }}
workingDirectory: '$(Build.BinariesDirectory)'

Expand Down
Loading

0 comments on commit 3df43a2

Please sign in to comment.