From b2356310f9a4dbe8630edf06598230ddffc0f4a9 Mon Sep 17 00:00:00 2001 From: Jonah Beckford <9566106-jonahbeckford@users.noreply.gitlab.com> Date: Tue, 28 Nov 2023 17:34:23 -0800 Subject: [PATCH] bump installer --- ci/setup-dkml/gh-darwin/post/action.yml | 32 +- ci/setup-dkml/gh-darwin/pre/action.yml | 140 ++++---- ci/setup-dkml/gh-linux/post/action.yml | 7 +- ci/setup-dkml/gh-linux/pre/action.yml | 118 +++---- ci/setup-dkml/gh-windows/post/action.yml | 7 +- ci/setup-dkml/gh-windows/pre/action.yml | 115 +++--- ci/setup-dkml/gl/setup-dkml.gitlab-ci.yml | 334 ++++++++---------- ci/setup-dkml/pc/setup-dkml-darwin_x86_64.sh | 125 +++---- ci/setup-dkml/pc/setup-dkml-linux_x86.sh | 116 +++--- ci/setup-dkml/pc/setup-dkml-linux_x86_64.sh | 116 +++--- ci/setup-dkml/pc/setup-dkml-windows_x86.ps1 | 119 +++---- .../pc/setup-dkml-windows_x86_64.ps1 | 119 +++---- i-network/bin/dune.inc | 50 ++- 13 files changed, 625 insertions(+), 773 deletions(-) diff --git a/ci/setup-dkml/gh-darwin/post/action.yml b/ci/setup-dkml/gh-darwin/post/action.yml index 0f0b2fc..fe9b246 100644 --- a/ci/setup-dkml/gh-darwin/post/action.yml +++ b/ci/setup-dkml/gh-darwin/post/action.yml @@ -18,20 +18,19 @@ runs: id: full_matrix_vars run: | # Select correct Darwin matrix variables - case "${{ matrix.dkml_host_abi }}" in - darwin_x86_64) - dkml_host_os='darwin'; - opam_root_cacheable='/Users/runner/.opam'; - abi_pattern='macos-darwin_all'; - gh_os='macos-latest'; - gh_unix_shell='sh'; - bootstrap_opam_version='2.2.0-alpha-20221228'; - dkml_host_abi='darwin_x86_64'; - opam_root='/Users/runner/.opam' ;; - - *) echo "FATAL: Unsupported dkml_host_abi=$dkml_host_abi in Darwin action.yml"; exit 107 ;; - esac + dkml_host_os='darwin' + opam_root_cacheable='/Users/runner/.opam' + abi_pattern='macos-darwin_all-intel' + no_gl='true' + gh_os='macos-latest' + gh_unix_shell='sh' + bootstrap_opam_version='2.2.0-alpha-20221228' + dkml_host_abi='darwin_x86_64' + opam_root='/Users/runner/.opam' + # + # note: All Darwin machines can cross-compile so they are equal except [dkml_host_abi]. + dkml_host_abi='${{ matrix.dkml_host_abi }}' add() { echo "$1=$2" | tee -a $GITHUB_OUTPUT | tee -a $GITHUB_ENV @@ -45,7 +44,7 @@ runs: add bootstrap_opam_version "${bootstrap_opam_version:-}" add ocaml_options "${ocaml_options:-}" - - name: Teardown DKML build apparatus + - name: Teardown DkML build apparatus shell: bash env: _STUB_FOR_AUTOGEN: "ON" # @@ -75,7 +74,6 @@ runs: PIN_DKML_BASE_COMPILER: '4.14.0~v2.1.0' PIN_DKML_BUILD_DESKTOP: '2.1.0' PIN_DKML_C_PROBE: '3.0.0' - PIN_DKML_COMPILER_ENV: '2.1.0' PIN_DKML_COMPILER_SRC: '2.1.0' PIN_DKML_COMPONENT_XX_CONSOLE: '0.1.1' PIN_DKML_EXE_LIB: '2.1.0' @@ -83,7 +81,7 @@ runs: PIN_DKML_INSTALL_INSTALLER: '0.5.2' PIN_DKML_INSTALL_RUNNER: '0.5.2' PIN_DKML_INSTALL: '0.5.2' - PIN_DKML_INSTALLER_OCAML_COMMON: '2.0.3' + PIN_DKML_INSTALLER_OCAML_COMMON: '2.1.0' PIN_DKML_PACKAGE_CONSOLE: '0.5.2' PIN_DKML_RUNTIME_COMMON_NATIVE: '2.1.0' PIN_DKML_RUNTIME_COMMON: '2.1.0' @@ -196,7 +194,7 @@ runs: PIN_UUTF: '1.0.3' PIN_WITH_DKML: '2.1.0' PIN_XDG: '3.9.0' - PIN_YOJSON: '2.1.1' + PIN_YOJSON: '2.1.2' PIN_ZED: '3.2.2' run: | sh .ci/sd4/run-teardown-dkml.sh GITHUB_WORKSPACE "$GITHUB_WORKSPACE" diff --git a/ci/setup-dkml/gh-darwin/pre/action.yml b/ci/setup-dkml/gh-darwin/pre/action.yml index 2e591e7..f07fb48 100644 --- a/ci/setup-dkml/gh-darwin/pre/action.yml +++ b/ci/setup-dkml/gh-darwin/pre/action.yml @@ -59,20 +59,19 @@ runs: id: full_matrix_vars run: | # Select correct Darwin matrix variables - case "${{ matrix.dkml_host_abi }}" in - darwin_x86_64) - dkml_host_os='darwin'; - opam_root_cacheable='/Users/runner/.opam'; - abi_pattern='macos-darwin_all'; - gh_os='macos-latest'; - gh_unix_shell='sh'; - bootstrap_opam_version='2.2.0-alpha-20221228'; - dkml_host_abi='darwin_x86_64'; - opam_root='/Users/runner/.opam' ;; - - *) echo "FATAL: Unsupported dkml_host_abi=$dkml_host_abi in Darwin action.yml"; exit 107 ;; - esac + dkml_host_os='darwin' + opam_root_cacheable='/Users/runner/.opam' + abi_pattern='macos-darwin_all-intel' + no_gl='true' + gh_os='macos-latest' + gh_unix_shell='sh' + bootstrap_opam_version='2.2.0-alpha-20221228' + dkml_host_abi='darwin_x86_64' + opam_root='/Users/runner/.opam' + # + # note: All Darwin machines can cross-compile so they are equal except [dkml_host_abi]. + dkml_host_abi='${{ matrix.dkml_host_abi }}' add() { echo "$1=$2" | tee -a $GITHUB_OUTPUT | tee -a $GITHUB_ENV @@ -229,7 +228,7 @@ runs: # dockcross (used by Linux) since a Docker-in-Docker container can have # difficulties doing a git checkout (the Git credentials for any private # repositories are likely not present). We don't care about any private - # repositories for DKML but any code that extends this (ex. DKSDK) may + # repositories for DkML but any code that extends this (ex. DKSDK) may # need to use private repositories. set -euf @@ -382,6 +381,33 @@ runs: # ------------------------------------------------------------------- + docker_fqin_preusername= # fully qualified image name (hostname[:port]/username/reponame[:tag]), the parts before the username (hostname[:port]/) + if [ -n "${docker_registry:-}" ]; then + docker_fqin_preusername="$docker_registry/" + fi + + # Extend dockcross. https://github.com/dockcross/dockcross#how-to-extend-dockcross-images + dockcross_image_id= + dockcross_cli_image_args= + if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then + echo "Doing docker build" + section_begin docker-build "Summary: docker build -t ${docker_fqin_preusername}dkml-workflows/dockcross" + + install -d .ci/sd4/docker-image + printf "FROM %s\nENV DEFAULT_DOCKCROSS_IMAGE %sdkml-workflows/dockcross:latest\nRUN if command -v apt-get; then apt-get install -y rsync %s && rm -rf /var/lib/apt/lists/*; fi\nRUN if command -v yum; then yum install -y rsync %s && yum clean all && rm -rf /var/cache/yum; fi" \ + "${dockcross_image:-}" "${docker_fqin_preusername}" "${dockcross_packages_apt:-}" "${dockcross_packages_yum:-}" >.ci/sd4/docker-image/Dockerfile + docker build -t "${docker_fqin_preusername}dkml-workflows/dockcross:latest" .ci/sd4/docker-image + + # Save image id to re-use for all remaining dockcross invocations + docker images --format "{{.ID}} {{.CreatedAt}}" | sort -rk 2 | awk 'NR==1{print $1}' | tee .ci/sd4/docker-image-id + dockcross_image_id=$(cat .ci/sd4/docker-image-id) + dockcross_cli_image_args="--image $dockcross_image_id" + + section_end docker-build + fi + + # ------------------------------------------------------------------- + section_begin setup-info "Summary: setup-dkml" SKIP_OPAM_MODIFICATIONS=${SKIP_OPAM_MODIFICATIONS:-false} # default is false @@ -432,8 +458,8 @@ runs: original_opam_root_cacheable=${original_opam_root_cacheable} unix_opam_root=${unix_opam_root} unix_opam_root_cacheable=${unix_opam_root_cacheable} + docker_registry=${docker_registry:-} dockcross_image=${dockcross_image:-} - dockcross_image_custom_prefix=${dockcross_image_custom_prefix:-} dockcross_run_extra_args=${dockcross_run_extra_args:-} docker_runner=${docker_runner:-} in_docker=${in_docker:-} @@ -593,12 +619,12 @@ runs: tar cf .ci/sd4/dist/run-with-env.tar -T /dev/null do_get_dockcross() { - if [ -n "${dockcross_image:-}" ]; then + if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then # The dockcross script is super-slow section_begin get-dockcross 'Get dockcross binary (ManyLinux)' install -d .ci/sd4 # shellcheck disable=SC2086 - docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_custom_prefix:-}${dockcross_image:-}" >.ci/sd4/dockcross.gen + docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_id}" >.ci/sd4/dockcross.gen # PROBLEM 1 # --------- @@ -696,10 +722,11 @@ runs: --rm \ \${ARGS:-} \ -v "\$HOST_PWD":/work \ - ${dockcross_image_custom_prefix:-}${dockcross_image:-} ${dockcross_entrypoint} "\$@" + "${dockcross_image_id}" ${dockcross_entrypoint} "\$@" else HERE=\$(dirname "\$0") HERE=\$(cd "\$HERE" && pwd) + export OCI_EXE=docker # default to podman if available, which breaks complaining about HTTPS vs HTTP on GitHub Actions communicating to http (docker) local registry. exec "\$HERE/dockcross-real" "\$@" fi EOF @@ -713,46 +740,6 @@ runs: } do_get_dockcross - if [ -n "${dockcross_image:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync ]; then - section_begin get-opam-prereqs-in-dockcross 'Get Opam prerequisites (ManyLinux)' - install -d .ci/sd4/bs/bin - # Install rsync with 'yum' (ManyLinux) or 'apt' (dockcross/linux-x64, etc.) - # if not present. - # shellcheck disable=SC2016 - .ci/sd4/dockcross --args "${dockcross_run_extra_args:-}" sh -c 'if ! command -v rsync; then if command -v yum; then sudo yum install -y rsync; else sudo apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync; fi; fi && install $(command -v rsync) .ci/sd4/bs/bin' - section_end get-opam-prereqs-in-dockcross - fi - fi - - # Opam prerequisites for using opam (not for installing opam) - - { - if [ -n "${docker_runner:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync.deps ]; then - section_begin get-opam-prereqs-in-docker 'Get Opam prerequisites (Linux Docker)' - install -d .ci/sd4/bs/bin - ${docker_runner} sh -c ' - apt-get update -qq -o=Dpkg::Use-Pty=0 && - apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync && - ldd /usr/bin/rsync && - ls -l /lib/i386-linux-gnu/libpopt.so.0 /lib/i386-linux-gnu/libacl.so.1 /lib/i386-linux-gnu/libattr.so.1 && - tar cCfhz / /work/.ci/sd4/bs/bin/deps.tar.gz /usr/bin/rsync /lib/i386-linux-gnu/libpopt.so.0 - ' - touch .ci/sd4/bs/bin/rsync.deps - section_end get-opam-prereqs-in-docker - fi - fi - - # Bundle Opam prerequisites (ManyLinux or Linux Docker) - if [ -n "${docker_runner:-}" ] || [ -n "${dockcross_image:-}" ]; then - # Bundle for consumers of setup-dkml.yml - do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/bs/bin/rsync - fi - } - # Get Opam Cache do_get_opam_cache() { if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi @@ -927,7 +914,7 @@ runs: termargs=-it fi - exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" + exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross ${dockcross_cli_image_args} --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" EOF chmod +x .ci/sd4/run-with-env @@ -939,7 +926,7 @@ runs: echo '___________________' >&2 do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/run-with-env .ci/sd4/run-in-docker .ci/sd4/edr - elif [ -n "${docker_runner:-}" ]; then + elif [ "${in_docker:-}" = "true" ] && [ -n "${docker_runner:-}" ]; then cat >.ci/sd4/run-with-env < .ci/sd4/cachekey.opam.binaries + echo 'b539a8-${{ inputs.FDOPEN_OPAMEXE_BOOTSTRAP }}-${{ steps.full_matrix_vars.outputs.dkml_host_abi }}-${{ steps.full_matrix_vars.outputs.opam_abi }}-${{ steps.full_matrix_vars.outputs.bootstrap_opam_version }}' > .ci/sd4/cachekey.opam.binaries echo '${{ steps.full_matrix_vars.outputs.abi_pattern }}-${{ steps.full_matrix_vars.outputs.vsstudio_arch }}-${{ steps.full_matrix_vars.outputs.vsstudio_hostarch }}-${{ steps.full_matrix_vars.outputs.vsstudio_dir }}-${{ steps.full_matrix_vars.outputs.vsstudio_vcvarsver }}-${{ steps.full_matrix_vars.outputs.vsstudio_winsdkver }}-${{ steps.full_matrix_vars.outputs.vsstudio_msvspreference }}-${{ steps.full_matrix_vars.outputs.vsstudio_cmakegenerator }}' > .ci/sd4/cachekey.vsstudio echo '${{ inputs.OCAML_COMPILER }}-${{ inputs.DISKUV_OPAM_REPOSITORY }}-${{ inputs.DKML_COMPILER }}-${{ inputs.CONF_DKML_CROSS_TOOLCHAIN }}' > .ci/sd4/cachekey.ci.inputs @@ -1789,7 +1774,7 @@ runs: # Checkout code - - name: Checkout DKML code + - name: Checkout DkML code shell: bash env: # Push down inputs variables for GitHub/GitLab portable scripts @@ -1838,7 +1823,6 @@ runs: PIN_DKML_BASE_COMPILER: '4.14.0~v2.1.0' PIN_DKML_BUILD_DESKTOP: '2.1.0' PIN_DKML_C_PROBE: '3.0.0' - PIN_DKML_COMPILER_ENV: '2.1.0' PIN_DKML_COMPILER_SRC: '2.1.0' PIN_DKML_COMPONENT_XX_CONSOLE: '0.1.1' PIN_DKML_EXE_LIB: '2.1.0' @@ -1846,7 +1830,7 @@ runs: PIN_DKML_INSTALL_INSTALLER: '0.5.2' PIN_DKML_INSTALL_RUNNER: '0.5.2' PIN_DKML_INSTALL: '0.5.2' - PIN_DKML_INSTALLER_OCAML_COMMON: '2.0.3' + PIN_DKML_INSTALLER_OCAML_COMMON: '2.1.0' PIN_DKML_PACKAGE_CONSOLE: '0.5.2' PIN_DKML_RUNTIME_COMMON_NATIVE: '2.1.0' PIN_DKML_RUNTIME_COMMON: '2.1.0' @@ -1959,7 +1943,7 @@ runs: PIN_UUTF: '1.0.3' PIN_WITH_DKML: '2.1.0' PIN_XDG: '3.9.0' - PIN_YOJSON: '2.1.1' + PIN_YOJSON: '2.1.2' PIN_ZED: '3.2.2' run: | sh .ci/sd4/run-setup-dkml.sh GITHUB_WORKSPACE "$GITHUB_WORKSPACE" diff --git a/ci/setup-dkml/gh-linux/post/action.yml b/ci/setup-dkml/gh-linux/post/action.yml index 0ae34d6..fe1e645 100644 --- a/ci/setup-dkml/gh-linux/post/action.yml +++ b/ci/setup-dkml/gh-linux/post/action.yml @@ -61,7 +61,7 @@ runs: add bootstrap_opam_version "${bootstrap_opam_version:-}" add ocaml_options "${ocaml_options:-}" - - name: Teardown DKML build apparatus + - name: Teardown DkML build apparatus shell: bash env: _STUB_FOR_AUTOGEN: "ON" # @@ -91,7 +91,6 @@ runs: PIN_DKML_BASE_COMPILER: '4.14.0~v2.1.0' PIN_DKML_BUILD_DESKTOP: '2.1.0' PIN_DKML_C_PROBE: '3.0.0' - PIN_DKML_COMPILER_ENV: '2.1.0' PIN_DKML_COMPILER_SRC: '2.1.0' PIN_DKML_COMPONENT_XX_CONSOLE: '0.1.1' PIN_DKML_EXE_LIB: '2.1.0' @@ -99,7 +98,7 @@ runs: PIN_DKML_INSTALL_INSTALLER: '0.5.2' PIN_DKML_INSTALL_RUNNER: '0.5.2' PIN_DKML_INSTALL: '0.5.2' - PIN_DKML_INSTALLER_OCAML_COMMON: '2.0.3' + PIN_DKML_INSTALLER_OCAML_COMMON: '2.1.0' PIN_DKML_PACKAGE_CONSOLE: '0.5.2' PIN_DKML_RUNTIME_COMMON_NATIVE: '2.1.0' PIN_DKML_RUNTIME_COMMON: '2.1.0' @@ -212,7 +211,7 @@ runs: PIN_UUTF: '1.0.3' PIN_WITH_DKML: '2.1.0' PIN_XDG: '3.9.0' - PIN_YOJSON: '2.1.1' + PIN_YOJSON: '2.1.2' PIN_ZED: '3.2.2' run: | sh .ci/sd4/run-teardown-dkml.sh GITHUB_WORKSPACE "$GITHUB_WORKSPACE" diff --git a/ci/setup-dkml/gh-linux/pre/action.yml b/ci/setup-dkml/gh-linux/pre/action.yml index 1352b2b..c7d1794 100644 --- a/ci/setup-dkml/gh-linux/pre/action.yml +++ b/ci/setup-dkml/gh-linux/pre/action.yml @@ -111,9 +111,6 @@ runs: add docker_runner "${docker_runner:-}" add in_docker "${in_docker:-}" - # GitHub Actions uses Docker Buildx, so use custom 'dkml-dockcross/*' cached image which speeds up builds - add dockcross_image_custom_prefix "dkml-" - - name: Write POSIX shell and AWK scripts shell: bash run: | @@ -257,7 +254,7 @@ runs: # dockcross (used by Linux) since a Docker-in-Docker container can have # difficulties doing a git checkout (the Git credentials for any private # repositories are likely not present). We don't care about any private - # repositories for DKML but any code that extends this (ex. DKSDK) may + # repositories for DkML but any code that extends this (ex. DKSDK) may # need to use private repositories. set -euf @@ -410,6 +407,33 @@ runs: # ------------------------------------------------------------------- + docker_fqin_preusername= # fully qualified image name (hostname[:port]/username/reponame[:tag]), the parts before the username (hostname[:port]/) + if [ -n "${docker_registry:-}" ]; then + docker_fqin_preusername="$docker_registry/" + fi + + # Extend dockcross. https://github.com/dockcross/dockcross#how-to-extend-dockcross-images + dockcross_image_id= + dockcross_cli_image_args= + if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then + echo "Doing docker build" + section_begin docker-build "Summary: docker build -t ${docker_fqin_preusername}dkml-workflows/dockcross" + + install -d .ci/sd4/docker-image + printf "FROM %s\nENV DEFAULT_DOCKCROSS_IMAGE %sdkml-workflows/dockcross:latest\nRUN if command -v apt-get; then apt-get install -y rsync %s && rm -rf /var/lib/apt/lists/*; fi\nRUN if command -v yum; then yum install -y rsync %s && yum clean all && rm -rf /var/cache/yum; fi" \ + "${dockcross_image:-}" "${docker_fqin_preusername}" "${dockcross_packages_apt:-}" "${dockcross_packages_yum:-}" >.ci/sd4/docker-image/Dockerfile + docker build -t "${docker_fqin_preusername}dkml-workflows/dockcross:latest" .ci/sd4/docker-image + + # Save image id to re-use for all remaining dockcross invocations + docker images --format "{{.ID}} {{.CreatedAt}}" | sort -rk 2 | awk 'NR==1{print $1}' | tee .ci/sd4/docker-image-id + dockcross_image_id=$(cat .ci/sd4/docker-image-id) + dockcross_cli_image_args="--image $dockcross_image_id" + + section_end docker-build + fi + + # ------------------------------------------------------------------- + section_begin setup-info "Summary: setup-dkml" SKIP_OPAM_MODIFICATIONS=${SKIP_OPAM_MODIFICATIONS:-false} # default is false @@ -460,8 +484,8 @@ runs: original_opam_root_cacheable=${original_opam_root_cacheable} unix_opam_root=${unix_opam_root} unix_opam_root_cacheable=${unix_opam_root_cacheable} + docker_registry=${docker_registry:-} dockcross_image=${dockcross_image:-} - dockcross_image_custom_prefix=${dockcross_image_custom_prefix:-} dockcross_run_extra_args=${dockcross_run_extra_args:-} docker_runner=${docker_runner:-} in_docker=${in_docker:-} @@ -621,12 +645,12 @@ runs: tar cf .ci/sd4/dist/run-with-env.tar -T /dev/null do_get_dockcross() { - if [ -n "${dockcross_image:-}" ]; then + if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then # The dockcross script is super-slow section_begin get-dockcross 'Get dockcross binary (ManyLinux)' install -d .ci/sd4 # shellcheck disable=SC2086 - docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_custom_prefix:-}${dockcross_image:-}" >.ci/sd4/dockcross.gen + docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_id}" >.ci/sd4/dockcross.gen # PROBLEM 1 # --------- @@ -724,10 +748,11 @@ runs: --rm \ \${ARGS:-} \ -v "\$HOST_PWD":/work \ - ${dockcross_image_custom_prefix:-}${dockcross_image:-} ${dockcross_entrypoint} "\$@" + "${dockcross_image_id}" ${dockcross_entrypoint} "\$@" else HERE=\$(dirname "\$0") HERE=\$(cd "\$HERE" && pwd) + export OCI_EXE=docker # default to podman if available, which breaks complaining about HTTPS vs HTTP on GitHub Actions communicating to http (docker) local registry. exec "\$HERE/dockcross-real" "\$@" fi EOF @@ -741,46 +766,6 @@ runs: } do_get_dockcross - if [ -n "${dockcross_image:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync ]; then - section_begin get-opam-prereqs-in-dockcross 'Get Opam prerequisites (ManyLinux)' - install -d .ci/sd4/bs/bin - # Install rsync with 'yum' (ManyLinux) or 'apt' (dockcross/linux-x64, etc.) - # if not present. - # shellcheck disable=SC2016 - .ci/sd4/dockcross --args "${dockcross_run_extra_args:-}" sh -c 'if ! command -v rsync; then if command -v yum; then sudo yum install -y rsync; else sudo apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync; fi; fi && install $(command -v rsync) .ci/sd4/bs/bin' - section_end get-opam-prereqs-in-dockcross - fi - fi - - # Opam prerequisites for using opam (not for installing opam) - - { - if [ -n "${docker_runner:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync.deps ]; then - section_begin get-opam-prereqs-in-docker 'Get Opam prerequisites (Linux Docker)' - install -d .ci/sd4/bs/bin - ${docker_runner} sh -c ' - apt-get update -qq -o=Dpkg::Use-Pty=0 && - apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync && - ldd /usr/bin/rsync && - ls -l /lib/i386-linux-gnu/libpopt.so.0 /lib/i386-linux-gnu/libacl.so.1 /lib/i386-linux-gnu/libattr.so.1 && - tar cCfhz / /work/.ci/sd4/bs/bin/deps.tar.gz /usr/bin/rsync /lib/i386-linux-gnu/libpopt.so.0 - ' - touch .ci/sd4/bs/bin/rsync.deps - section_end get-opam-prereqs-in-docker - fi - fi - - # Bundle Opam prerequisites (ManyLinux or Linux Docker) - if [ -n "${docker_runner:-}" ] || [ -n "${dockcross_image:-}" ]; then - # Bundle for consumers of setup-dkml.yml - do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/bs/bin/rsync - fi - } - # Get Opam Cache do_get_opam_cache() { if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi @@ -955,7 +940,7 @@ runs: termargs=-it fi - exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" + exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross ${dockcross_cli_image_args} --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" EOF chmod +x .ci/sd4/run-with-env @@ -967,7 +952,7 @@ runs: echo '___________________' >&2 do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/run-with-env .ci/sd4/run-in-docker .ci/sd4/edr - elif [ -n "${docker_runner:-}" ]; then + elif [ "${in_docker:-}" = "true" ] && [ -n "${docker_runner:-}" ]; then cat >.ci/sd4/run-with-env < .ci/sd4/cachekey.opam.binaries + echo 'b539a8-${{ inputs.FDOPEN_OPAMEXE_BOOTSTRAP }}-${{ steps.full_matrix_vars.outputs.dkml_host_abi }}-${{ steps.full_matrix_vars.outputs.opam_abi }}-${{ steps.full_matrix_vars.outputs.bootstrap_opam_version }}' > .ci/sd4/cachekey.opam.binaries echo '${{ steps.full_matrix_vars.outputs.abi_pattern }}-${{ steps.full_matrix_vars.outputs.vsstudio_arch }}-${{ steps.full_matrix_vars.outputs.vsstudio_hostarch }}-${{ steps.full_matrix_vars.outputs.vsstudio_dir }}-${{ steps.full_matrix_vars.outputs.vsstudio_vcvarsver }}-${{ steps.full_matrix_vars.outputs.vsstudio_winsdkver }}-${{ steps.full_matrix_vars.outputs.vsstudio_msvspreference }}-${{ steps.full_matrix_vars.outputs.vsstudio_cmakegenerator }}' > .ci/sd4/cachekey.vsstudio echo '${{ inputs.OCAML_COMPILER }}-${{ inputs.DISKUV_OPAM_REPOSITORY }}-${{ inputs.DKML_COMPILER }}-${{ inputs.CONF_DKML_CROSS_TOOLCHAIN }}' > .ci/sd4/cachekey.ci.inputs @@ -1847,7 +1830,7 @@ runs: # Checkout code - - name: Checkout DKML code + - name: Checkout DkML code shell: bash env: # Push down inputs variables for GitHub/GitLab portable scripts @@ -1897,7 +1880,6 @@ runs: PIN_DKML_BASE_COMPILER: '4.14.0~v2.1.0' PIN_DKML_BUILD_DESKTOP: '2.1.0' PIN_DKML_C_PROBE: '3.0.0' - PIN_DKML_COMPILER_ENV: '2.1.0' PIN_DKML_COMPILER_SRC: '2.1.0' PIN_DKML_COMPONENT_XX_CONSOLE: '0.1.1' PIN_DKML_EXE_LIB: '2.1.0' @@ -1905,7 +1887,7 @@ runs: PIN_DKML_INSTALL_INSTALLER: '0.5.2' PIN_DKML_INSTALL_RUNNER: '0.5.2' PIN_DKML_INSTALL: '0.5.2' - PIN_DKML_INSTALLER_OCAML_COMMON: '2.0.3' + PIN_DKML_INSTALLER_OCAML_COMMON: '2.1.0' PIN_DKML_PACKAGE_CONSOLE: '0.5.2' PIN_DKML_RUNTIME_COMMON_NATIVE: '2.1.0' PIN_DKML_RUNTIME_COMMON: '2.1.0' @@ -2018,7 +2000,7 @@ runs: PIN_UUTF: '1.0.3' PIN_WITH_DKML: '2.1.0' PIN_XDG: '3.9.0' - PIN_YOJSON: '2.1.1' + PIN_YOJSON: '2.1.2' PIN_ZED: '3.2.2' run: | sh .ci/sd4/run-setup-dkml.sh GITHUB_WORKSPACE "$GITHUB_WORKSPACE" diff --git a/ci/setup-dkml/gh-windows/post/action.yml b/ci/setup-dkml/gh-windows/post/action.yml index a30ef60..9ac537c 100644 --- a/ci/setup-dkml/gh-windows/post/action.yml +++ b/ci/setup-dkml/gh-windows/post/action.yml @@ -74,7 +74,7 @@ runs: add bootstrap_opam_version "${bootstrap_opam_version:-}" add ocaml_options "${ocaml_options:-}" - - name: Teardown DKML build apparatus + - name: Teardown DkML build apparatus shell: msys2 {0} env: _STUB_FOR_AUTOGEN: "ON" # @@ -104,7 +104,6 @@ runs: PIN_DKML_BASE_COMPILER: '4.14.0~v2.1.0' PIN_DKML_BUILD_DESKTOP: '2.1.0' PIN_DKML_C_PROBE: '3.0.0' - PIN_DKML_COMPILER_ENV: '2.1.0' PIN_DKML_COMPILER_SRC: '2.1.0' PIN_DKML_COMPONENT_XX_CONSOLE: '0.1.1' PIN_DKML_EXE_LIB: '2.1.0' @@ -112,7 +111,7 @@ runs: PIN_DKML_INSTALL_INSTALLER: '0.5.2' PIN_DKML_INSTALL_RUNNER: '0.5.2' PIN_DKML_INSTALL: '0.5.2' - PIN_DKML_INSTALLER_OCAML_COMMON: '2.0.3' + PIN_DKML_INSTALLER_OCAML_COMMON: '2.1.0' PIN_DKML_PACKAGE_CONSOLE: '0.5.2' PIN_DKML_RUNTIME_COMMON_NATIVE: '2.1.0' PIN_DKML_RUNTIME_COMMON: '2.1.0' @@ -225,7 +224,7 @@ runs: PIN_UUTF: '1.0.3' PIN_WITH_DKML: '2.1.0' PIN_XDG: '3.9.0' - PIN_YOJSON: '2.1.1' + PIN_YOJSON: '2.1.2' PIN_ZED: '3.2.2' run: | sh .ci/sd4/run-teardown-dkml.sh GITHUB_WORKSPACE "$GITHUB_WORKSPACE" diff --git a/ci/setup-dkml/gh-windows/pre/action.yml b/ci/setup-dkml/gh-windows/pre/action.yml index 06c8f48..7109253 100644 --- a/ci/setup-dkml/gh-windows/pre/action.yml +++ b/ci/setup-dkml/gh-windows/pre/action.yml @@ -290,7 +290,7 @@ runs: # dockcross (used by Linux) since a Docker-in-Docker container can have # difficulties doing a git checkout (the Git credentials for any private # repositories are likely not present). We don't care about any private - # repositories for DKML but any code that extends this (ex. DKSDK) may + # repositories for DkML but any code that extends this (ex. DKSDK) may # need to use private repositories. set -euf @@ -443,6 +443,33 @@ runs: # ------------------------------------------------------------------- + docker_fqin_preusername= # fully qualified image name (hostname[:port]/username/reponame[:tag]), the parts before the username (hostname[:port]/) + if [ -n "${docker_registry:-}" ]; then + docker_fqin_preusername="$docker_registry/" + fi + + # Extend dockcross. https://github.com/dockcross/dockcross#how-to-extend-dockcross-images + dockcross_image_id= + dockcross_cli_image_args= + if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then + echo "Doing docker build" + section_begin docker-build "Summary: docker build -t ${docker_fqin_preusername}dkml-workflows/dockcross" + + install -d .ci/sd4/docker-image + printf "FROM %s\nENV DEFAULT_DOCKCROSS_IMAGE %sdkml-workflows/dockcross:latest\nRUN if command -v apt-get; then apt-get install -y rsync %s && rm -rf /var/lib/apt/lists/*; fi\nRUN if command -v yum; then yum install -y rsync %s && yum clean all && rm -rf /var/cache/yum; fi" \ + "${dockcross_image:-}" "${docker_fqin_preusername}" "${dockcross_packages_apt:-}" "${dockcross_packages_yum:-}" >.ci/sd4/docker-image/Dockerfile + docker build -t "${docker_fqin_preusername}dkml-workflows/dockcross:latest" .ci/sd4/docker-image + + # Save image id to re-use for all remaining dockcross invocations + docker images --format "{{.ID}} {{.CreatedAt}}" | sort -rk 2 | awk 'NR==1{print $1}' | tee .ci/sd4/docker-image-id + dockcross_image_id=$(cat .ci/sd4/docker-image-id) + dockcross_cli_image_args="--image $dockcross_image_id" + + section_end docker-build + fi + + # ------------------------------------------------------------------- + section_begin setup-info "Summary: setup-dkml" SKIP_OPAM_MODIFICATIONS=${SKIP_OPAM_MODIFICATIONS:-false} # default is false @@ -493,8 +520,8 @@ runs: original_opam_root_cacheable=${original_opam_root_cacheable} unix_opam_root=${unix_opam_root} unix_opam_root_cacheable=${unix_opam_root_cacheable} + docker_registry=${docker_registry:-} dockcross_image=${dockcross_image:-} - dockcross_image_custom_prefix=${dockcross_image_custom_prefix:-} dockcross_run_extra_args=${dockcross_run_extra_args:-} docker_runner=${docker_runner:-} in_docker=${in_docker:-} @@ -654,12 +681,12 @@ runs: tar cf .ci/sd4/dist/run-with-env.tar -T /dev/null do_get_dockcross() { - if [ -n "${dockcross_image:-}" ]; then + if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then # The dockcross script is super-slow section_begin get-dockcross 'Get dockcross binary (ManyLinux)' install -d .ci/sd4 # shellcheck disable=SC2086 - docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_custom_prefix:-}${dockcross_image:-}" >.ci/sd4/dockcross.gen + docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_id}" >.ci/sd4/dockcross.gen # PROBLEM 1 # --------- @@ -757,10 +784,11 @@ runs: --rm \ \${ARGS:-} \ -v "\$HOST_PWD":/work \ - ${dockcross_image_custom_prefix:-}${dockcross_image:-} ${dockcross_entrypoint} "\$@" + "${dockcross_image_id}" ${dockcross_entrypoint} "\$@" else HERE=\$(dirname "\$0") HERE=\$(cd "\$HERE" && pwd) + export OCI_EXE=docker # default to podman if available, which breaks complaining about HTTPS vs HTTP on GitHub Actions communicating to http (docker) local registry. exec "\$HERE/dockcross-real" "\$@" fi EOF @@ -774,46 +802,6 @@ runs: } do_get_dockcross - if [ -n "${dockcross_image:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync ]; then - section_begin get-opam-prereqs-in-dockcross 'Get Opam prerequisites (ManyLinux)' - install -d .ci/sd4/bs/bin - # Install rsync with 'yum' (ManyLinux) or 'apt' (dockcross/linux-x64, etc.) - # if not present. - # shellcheck disable=SC2016 - .ci/sd4/dockcross --args "${dockcross_run_extra_args:-}" sh -c 'if ! command -v rsync; then if command -v yum; then sudo yum install -y rsync; else sudo apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync; fi; fi && install $(command -v rsync) .ci/sd4/bs/bin' - section_end get-opam-prereqs-in-dockcross - fi - fi - - # Opam prerequisites for using opam (not for installing opam) - - { - if [ -n "${docker_runner:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync.deps ]; then - section_begin get-opam-prereqs-in-docker 'Get Opam prerequisites (Linux Docker)' - install -d .ci/sd4/bs/bin - ${docker_runner} sh -c ' - apt-get update -qq -o=Dpkg::Use-Pty=0 && - apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync && - ldd /usr/bin/rsync && - ls -l /lib/i386-linux-gnu/libpopt.so.0 /lib/i386-linux-gnu/libacl.so.1 /lib/i386-linux-gnu/libattr.so.1 && - tar cCfhz / /work/.ci/sd4/bs/bin/deps.tar.gz /usr/bin/rsync /lib/i386-linux-gnu/libpopt.so.0 - ' - touch .ci/sd4/bs/bin/rsync.deps - section_end get-opam-prereqs-in-docker - fi - fi - - # Bundle Opam prerequisites (ManyLinux or Linux Docker) - if [ -n "${docker_runner:-}" ] || [ -n "${dockcross_image:-}" ]; then - # Bundle for consumers of setup-dkml.yml - do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/bs/bin/rsync - fi - } - # Get Opam Cache do_get_opam_cache() { if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi @@ -988,7 +976,7 @@ runs: termargs=-it fi - exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" + exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross ${dockcross_cli_image_args} --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" EOF chmod +x .ci/sd4/run-with-env @@ -1000,7 +988,7 @@ runs: echo '___________________' >&2 do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/run-with-env .ci/sd4/run-in-docker .ci/sd4/edr - elif [ -n "${docker_runner:-}" ]; then + elif [ "${in_docker:-}" = "true" ] && [ -n "${docker_runner:-}" ]; then cat >.ci/sd4/run-with-env < .ci/sd4/cachekey.opam.binaries + echo 'b539a8-${{ inputs.FDOPEN_OPAMEXE_BOOTSTRAP }}-${{ steps.full_matrix_vars.outputs.dkml_host_abi }}-${{ steps.full_matrix_vars.outputs.opam_abi }}-${{ steps.full_matrix_vars.outputs.bootstrap_opam_version }}' > .ci/sd4/cachekey.opam.binaries echo '${{ steps.full_matrix_vars.outputs.abi_pattern }}-${{ steps.full_matrix_vars.outputs.vsstudio_arch }}-${{ steps.full_matrix_vars.outputs.vsstudio_hostarch }}-${{ steps.full_matrix_vars.outputs.vsstudio_dir }}-${{ steps.full_matrix_vars.outputs.vsstudio_vcvarsver }}-${{ steps.full_matrix_vars.outputs.vsstudio_winsdkver }}-${{ steps.full_matrix_vars.outputs.vsstudio_msvspreference }}-${{ steps.full_matrix_vars.outputs.vsstudio_cmakegenerator }}' > .ci/sd4/cachekey.vsstudio echo '${{ inputs.OCAML_COMPILER }}-${{ inputs.DISKUV_OPAM_REPOSITORY }}-${{ inputs.DKML_COMPILER }}-${{ inputs.CONF_DKML_CROSS_TOOLCHAIN }}' > .ci/sd4/cachekey.ci.inputs @@ -1988,7 +1974,7 @@ runs: # Checkout code - - name: Checkout DKML code + - name: Checkout DkML code shell: bash env: # Push down inputs variables for GitHub/GitLab portable scripts @@ -2069,7 +2055,6 @@ runs: PIN_DKML_BASE_COMPILER: '4.14.0~v2.1.0' PIN_DKML_BUILD_DESKTOP: '2.1.0' PIN_DKML_C_PROBE: '3.0.0' - PIN_DKML_COMPILER_ENV: '2.1.0' PIN_DKML_COMPILER_SRC: '2.1.0' PIN_DKML_COMPONENT_XX_CONSOLE: '0.1.1' PIN_DKML_EXE_LIB: '2.1.0' @@ -2077,7 +2062,7 @@ runs: PIN_DKML_INSTALL_INSTALLER: '0.5.2' PIN_DKML_INSTALL_RUNNER: '0.5.2' PIN_DKML_INSTALL: '0.5.2' - PIN_DKML_INSTALLER_OCAML_COMMON: '2.0.3' + PIN_DKML_INSTALLER_OCAML_COMMON: '2.1.0' PIN_DKML_PACKAGE_CONSOLE: '0.5.2' PIN_DKML_RUNTIME_COMMON_NATIVE: '2.1.0' PIN_DKML_RUNTIME_COMMON: '2.1.0' @@ -2190,7 +2175,7 @@ runs: PIN_UUTF: '1.0.3' PIN_WITH_DKML: '2.1.0' PIN_XDG: '3.9.0' - PIN_YOJSON: '2.1.1' + PIN_YOJSON: '2.1.2' PIN_ZED: '3.2.2' run: | sh .ci/sd4/run-setup-dkml.sh GITHUB_WORKSPACE "$GITHUB_WORKSPACE" diff --git a/ci/setup-dkml/gl/setup-dkml.gitlab-ci.yml b/ci/setup-dkml/gl/setup-dkml.gitlab-ci.yml index ec1ae2f..bb3ef60 100644 --- a/ci/setup-dkml/gl/setup-dkml.gitlab-ci.yml +++ b/ci/setup-dkml/gl/setup-dkml.gitlab-ci.yml @@ -40,7 +40,6 @@ variables: PIN_DKML_BASE_COMPILER: '4.14.0~v2.1.0' PIN_DKML_BUILD_DESKTOP: '2.1.0' PIN_DKML_C_PROBE: '3.0.0' - PIN_DKML_COMPILER_ENV: '2.1.0' PIN_DKML_COMPILER_SRC: '2.1.0' PIN_DKML_COMPONENT_XX_CONSOLE: '0.1.1' PIN_DKML_EXE_LIB: '2.1.0' @@ -48,7 +47,7 @@ variables: PIN_DKML_INSTALL_INSTALLER: '0.5.2' PIN_DKML_INSTALL_RUNNER: '0.5.2' PIN_DKML_INSTALL: '0.5.2' - PIN_DKML_INSTALLER_OCAML_COMMON: '2.0.3' + PIN_DKML_INSTALLER_OCAML_COMMON: '2.1.0' PIN_DKML_PACKAGE_CONSOLE: '0.5.2' PIN_DKML_RUNTIME_COMMON_NATIVE: '2.1.0' PIN_DKML_RUNTIME_COMMON: '2.1.0' @@ -161,7 +160,7 @@ variables: PIN_UUTF: '1.0.3' PIN_WITH_DKML: '2.1.0' PIN_XDG: '3.9.0' - PIN_YOJSON: '2.1.1' + PIN_YOJSON: '2.1.2' PIN_ZED: '3.2.2' .common:setup-dkml-allplusmincache: @@ -174,14 +173,14 @@ variables: expire_in: "1 days" # Only 4 caches allowed in GitLab CI cache: - - key: "a2_1_${CACHE_PREFIX}-28c7f18-${abi_pattern}-${CI_COMMIT_REF_SLUG}" + - key: "a2_1_${CACHE_PREFIX}-db62c42-${abi_pattern}-${CI_COMMIT_REF_SLUG}" paths: - ${opam_root_cacheable}/download-cache .common:setup-dkml-cache: # Only 4 caches allowed in GitLab CI cache: - - key: "b2_1_${CACHE_PREFIX}-28c7f18-${abi_pattern}-${CI_COMMIT_REF_SLUG}" + - key: "b2_1_${CACHE_PREFIX}-db62c42-${abi_pattern}-${CI_COMMIT_REF_SLUG}" paths: - .ci/sd4/bs - ${opam_root_cacheable}/config @@ -486,7 +485,7 @@ variables: # dockcross (used by Linux) since a Docker-in-Docker container can have # difficulties doing a git checkout (the Git credentials for any private # repositories are likely not present). We don't care about any private - # repositories for DKML but any code that extends this (ex. DKSDK) may + # repositories for DkML but any code that extends this (ex. DKSDK) may # need to use private repositories. set -euf @@ -642,6 +641,33 @@ variables: # ------------------------------------------------------------------- + docker_fqin_preusername= # fully qualified image name (hostname[:port]/username/reponame[:tag]), the parts before the username (hostname[:port]/) + if [ -n "${docker_registry:-}" ]; then + docker_fqin_preusername="$docker_registry/" + fi + + # Extend dockcross. https://github.com/dockcross/dockcross#how-to-extend-dockcross-images + dockcross_image_id= + dockcross_cli_image_args= + if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then + echo "Doing docker build" + section_begin docker-build "Summary: docker build -t ${docker_fqin_preusername}dkml-workflows/dockcross" + + install -d .ci/sd4/docker-image + printf "FROM %s\nENV DEFAULT_DOCKCROSS_IMAGE %sdkml-workflows/dockcross:latest\nRUN if command -v apt-get; then apt-get install -y rsync %s && rm -rf /var/lib/apt/lists/*; fi\nRUN if command -v yum; then yum install -y rsync %s && yum clean all && rm -rf /var/cache/yum; fi" \ + "${dockcross_image:-}" "${docker_fqin_preusername}" "${dockcross_packages_apt:-}" "${dockcross_packages_yum:-}" >.ci/sd4/docker-image/Dockerfile + docker build -t "${docker_fqin_preusername}dkml-workflows/dockcross:latest" .ci/sd4/docker-image + + # Save image id to re-use for all remaining dockcross invocations + docker images --format "{{.ID}} {{.CreatedAt}}" | sort -rk 2 | awk 'NR==1{print $1}' | tee .ci/sd4/docker-image-id + dockcross_image_id=$(cat .ci/sd4/docker-image-id) + dockcross_cli_image_args="--image $dockcross_image_id" + + section_end docker-build + fi + + # ------------------------------------------------------------------- + section_begin setup-info "Summary: setup-dkml" SKIP_OPAM_MODIFICATIONS=${SKIP_OPAM_MODIFICATIONS:-false} # default is false @@ -692,8 +718,8 @@ variables: original_opam_root_cacheable=${original_opam_root_cacheable} unix_opam_root=${unix_opam_root} unix_opam_root_cacheable=${unix_opam_root_cacheable} + docker_registry=${docker_registry:-} dockcross_image=${dockcross_image:-} - dockcross_image_custom_prefix=${dockcross_image_custom_prefix:-} dockcross_run_extra_args=${dockcross_run_extra_args:-} docker_runner=${docker_runner:-} in_docker=${in_docker:-} @@ -853,12 +879,12 @@ variables: tar cf .ci/sd4/dist/run-with-env.tar -T /dev/null do_get_dockcross() { - if [ -n "${dockcross_image:-}" ]; then + if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then # The dockcross script is super-slow section_begin get-dockcross 'Get dockcross binary (ManyLinux)' install -d .ci/sd4 # shellcheck disable=SC2086 - docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_custom_prefix:-}${dockcross_image:-}" >.ci/sd4/dockcross.gen + docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_id}" >.ci/sd4/dockcross.gen # PROBLEM 1 # --------- @@ -956,10 +982,11 @@ variables: --rm \ \${ARGS:-} \ -v "\$HOST_PWD":/work \ - ${dockcross_image_custom_prefix:-}${dockcross_image:-} ${dockcross_entrypoint} "\$@" + "${dockcross_image_id}" ${dockcross_entrypoint} "\$@" else HERE=\$(dirname "\$0") HERE=\$(cd "\$HERE" && pwd) + export OCI_EXE=docker # default to podman if available, which breaks complaining about HTTPS vs HTTP on GitHub Actions communicating to http (docker) local registry. exec "\$HERE/dockcross-real" "\$@" fi EOF @@ -973,46 +1000,6 @@ variables: } do_get_dockcross - if [ -n "${dockcross_image:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync ]; then - section_begin get-opam-prereqs-in-dockcross 'Get Opam prerequisites (ManyLinux)' - install -d .ci/sd4/bs/bin - # Install rsync with 'yum' (ManyLinux) or 'apt' (dockcross/linux-x64, etc.) - # if not present. - # shellcheck disable=SC2016 - .ci/sd4/dockcross --args "${dockcross_run_extra_args:-}" sh -c 'if ! command -v rsync; then if command -v yum; then sudo yum install -y rsync; else sudo apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync; fi; fi && install $(command -v rsync) .ci/sd4/bs/bin' - section_end get-opam-prereqs-in-dockcross - fi - fi - - # Opam prerequisites for using opam (not for installing opam) - - { - if [ -n "${docker_runner:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync.deps ]; then - section_begin get-opam-prereqs-in-docker 'Get Opam prerequisites (Linux Docker)' - install -d .ci/sd4/bs/bin - ${docker_runner} sh -c ' - apt-get update -qq -o=Dpkg::Use-Pty=0 && - apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync && - ldd /usr/bin/rsync && - ls -l /lib/i386-linux-gnu/libpopt.so.0 /lib/i386-linux-gnu/libacl.so.1 /lib/i386-linux-gnu/libattr.so.1 && - tar cCfhz / /work/.ci/sd4/bs/bin/deps.tar.gz /usr/bin/rsync /lib/i386-linux-gnu/libpopt.so.0 - ' - touch .ci/sd4/bs/bin/rsync.deps - section_end get-opam-prereqs-in-docker - fi - fi - - # Bundle Opam prerequisites (ManyLinux or Linux Docker) - if [ -n "${docker_runner:-}" ] || [ -n "${dockcross_image:-}" ]; then - # Bundle for consumers of setup-dkml.yml - do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/bs/bin/rsync - fi - } - # Get Opam Cache do_get_opam_cache() { if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi @@ -1187,7 +1174,7 @@ variables: termargs=-it fi - exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" + exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross ${dockcross_cli_image_args} --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" EOF chmod +x .ci/sd4/run-with-env @@ -1199,7 +1186,7 @@ variables: echo '___________________' >&2 do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/run-with-env .ci/sd4/run-in-docker .ci/sd4/edr - elif [ -n "${docker_runner:-}" ]; then + elif [ "${in_docker:-}" = "true" ] && [ -n "${docker_runner:-}" ]; then cat >.ci/sd4/run-with-env <.ci/sd4/docker-image/Dockerfile + docker build -t "${docker_fqin_preusername}dkml-workflows/dockcross:latest" .ci/sd4/docker-image + + # Save image id to re-use for all remaining dockcross invocations + docker images --format "{{.ID}} {{.CreatedAt}}" | sort -rk 2 | awk 'NR==1{print $1}' | tee .ci/sd4/docker-image-id + dockcross_image_id=$(cat .ci/sd4/docker-image-id) + dockcross_cli_image_args="--image $dockcross_image_id" + + section_end docker-build + fi + + # ------------------------------------------------------------------- + section_begin setup-info "Summary: setup-dkml" SKIP_OPAM_MODIFICATIONS=${SKIP_OPAM_MODIFICATIONS:-false} # default is false @@ -2659,8 +2672,8 @@ variables: original_opam_root_cacheable=${original_opam_root_cacheable} unix_opam_root=${unix_opam_root} unix_opam_root_cacheable=${unix_opam_root_cacheable} + docker_registry=${docker_registry:-} dockcross_image=${dockcross_image:-} - dockcross_image_custom_prefix=${dockcross_image_custom_prefix:-} dockcross_run_extra_args=${dockcross_run_extra_args:-} docker_runner=${docker_runner:-} in_docker=${in_docker:-} @@ -2820,12 +2833,12 @@ variables: tar cf .ci/sd4/dist/run-with-env.tar -T /dev/null do_get_dockcross() { - if [ -n "${dockcross_image:-}" ]; then + if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then # The dockcross script is super-slow section_begin get-dockcross 'Get dockcross binary (ManyLinux)' install -d .ci/sd4 # shellcheck disable=SC2086 - docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_custom_prefix:-}${dockcross_image:-}" >.ci/sd4/dockcross.gen + docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_id}" >.ci/sd4/dockcross.gen # PROBLEM 1 # --------- @@ -2923,10 +2936,11 @@ variables: --rm \ \${ARGS:-} \ -v "\$HOST_PWD":/work \ - ${dockcross_image_custom_prefix:-}${dockcross_image:-} ${dockcross_entrypoint} "\$@" + "${dockcross_image_id}" ${dockcross_entrypoint} "\$@" else HERE=\$(dirname "\$0") HERE=\$(cd "\$HERE" && pwd) + export OCI_EXE=docker # default to podman if available, which breaks complaining about HTTPS vs HTTP on GitHub Actions communicating to http (docker) local registry. exec "\$HERE/dockcross-real" "\$@" fi EOF @@ -2940,46 +2954,6 @@ variables: } do_get_dockcross - if [ -n "${dockcross_image:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync ]; then - section_begin get-opam-prereqs-in-dockcross 'Get Opam prerequisites (ManyLinux)' - install -d .ci/sd4/bs/bin - # Install rsync with 'yum' (ManyLinux) or 'apt' (dockcross/linux-x64, etc.) - # if not present. - # shellcheck disable=SC2016 - .ci/sd4/dockcross --args "${dockcross_run_extra_args:-}" sh -c 'if ! command -v rsync; then if command -v yum; then sudo yum install -y rsync; else sudo apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync; fi; fi && install $(command -v rsync) .ci/sd4/bs/bin' - section_end get-opam-prereqs-in-dockcross - fi - fi - - # Opam prerequisites for using opam (not for installing opam) - - { - if [ -n "${docker_runner:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync.deps ]; then - section_begin get-opam-prereqs-in-docker 'Get Opam prerequisites (Linux Docker)' - install -d .ci/sd4/bs/bin - ${docker_runner} sh -c ' - apt-get update -qq -o=Dpkg::Use-Pty=0 && - apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync && - ldd /usr/bin/rsync && - ls -l /lib/i386-linux-gnu/libpopt.so.0 /lib/i386-linux-gnu/libacl.so.1 /lib/i386-linux-gnu/libattr.so.1 && - tar cCfhz / /work/.ci/sd4/bs/bin/deps.tar.gz /usr/bin/rsync /lib/i386-linux-gnu/libpopt.so.0 - ' - touch .ci/sd4/bs/bin/rsync.deps - section_end get-opam-prereqs-in-docker - fi - fi - - # Bundle Opam prerequisites (ManyLinux or Linux Docker) - if [ -n "${docker_runner:-}" ] || [ -n "${dockcross_image:-}" ]; then - # Bundle for consumers of setup-dkml.yml - do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/bs/bin/rsync - fi - } - # Get Opam Cache do_get_opam_cache() { if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi @@ -3154,7 +3128,7 @@ variables: termargs=-it fi - exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" + exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross ${dockcross_cli_image_args} --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" EOF chmod +x .ci/sd4/run-with-env @@ -3166,7 +3140,7 @@ variables: echo '___________________' >&2 do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/run-with-env .ci/sd4/run-in-docker .ci/sd4/edr - elif [ -n "${docker_runner:-}" ]; then + elif [ "${in_docker:-}" = "true" ] && [ -n "${docker_runner:-}" ]; then cat >.ci/sd4/run-with-env <.ci/sd4/docker-image/Dockerfile + docker build -t "${docker_fqin_preusername}dkml-workflows/dockcross:latest" .ci/sd4/docker-image + + # Save image id to re-use for all remaining dockcross invocations + docker images --format "{{.ID}} {{.CreatedAt}}" | sort -rk 2 | awk 'NR==1{print $1}' | tee .ci/sd4/docker-image-id + dockcross_image_id=$(cat .ci/sd4/docker-image-id) + dockcross_cli_image_args="--image $dockcross_image_id" + + section_end docker-build + fi + + # ------------------------------------------------------------------- + section_begin setup-info "Summary: setup-dkml" SKIP_OPAM_MODIFICATIONS=${SKIP_OPAM_MODIFICATIONS:-false} # default is false @@ -4344,8 +4343,8 @@ variables: original_opam_root_cacheable=${original_opam_root_cacheable} unix_opam_root=${unix_opam_root} unix_opam_root_cacheable=${unix_opam_root_cacheable} + docker_registry=${docker_registry:-} dockcross_image=${dockcross_image:-} - dockcross_image_custom_prefix=${dockcross_image_custom_prefix:-} dockcross_run_extra_args=${dockcross_run_extra_args:-} docker_runner=${docker_runner:-} in_docker=${in_docker:-} @@ -4505,12 +4504,12 @@ variables: tar cf .ci/sd4/dist/run-with-env.tar -T /dev/null do_get_dockcross() { - if [ -n "${dockcross_image:-}" ]; then + if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then # The dockcross script is super-slow section_begin get-dockcross 'Get dockcross binary (ManyLinux)' install -d .ci/sd4 # shellcheck disable=SC2086 - docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_custom_prefix:-}${dockcross_image:-}" >.ci/sd4/dockcross.gen + docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_id}" >.ci/sd4/dockcross.gen # PROBLEM 1 # --------- @@ -4608,10 +4607,11 @@ variables: --rm \ \${ARGS:-} \ -v "\$HOST_PWD":/work \ - ${dockcross_image_custom_prefix:-}${dockcross_image:-} ${dockcross_entrypoint} "\$@" + "${dockcross_image_id}" ${dockcross_entrypoint} "\$@" else HERE=\$(dirname "\$0") HERE=\$(cd "\$HERE" && pwd) + export OCI_EXE=docker # default to podman if available, which breaks complaining about HTTPS vs HTTP on GitHub Actions communicating to http (docker) local registry. exec "\$HERE/dockcross-real" "\$@" fi EOF @@ -4625,46 +4625,6 @@ variables: } do_get_dockcross - if [ -n "${dockcross_image:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync ]; then - section_begin get-opam-prereqs-in-dockcross 'Get Opam prerequisites (ManyLinux)' - install -d .ci/sd4/bs/bin - # Install rsync with 'yum' (ManyLinux) or 'apt' (dockcross/linux-x64, etc.) - # if not present. - # shellcheck disable=SC2016 - .ci/sd4/dockcross --args "${dockcross_run_extra_args:-}" sh -c 'if ! command -v rsync; then if command -v yum; then sudo yum install -y rsync; else sudo apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync; fi; fi && install $(command -v rsync) .ci/sd4/bs/bin' - section_end get-opam-prereqs-in-dockcross - fi - fi - - # Opam prerequisites for using opam (not for installing opam) - - { - if [ -n "${docker_runner:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync.deps ]; then - section_begin get-opam-prereqs-in-docker 'Get Opam prerequisites (Linux Docker)' - install -d .ci/sd4/bs/bin - ${docker_runner} sh -c ' - apt-get update -qq -o=Dpkg::Use-Pty=0 && - apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync && - ldd /usr/bin/rsync && - ls -l /lib/i386-linux-gnu/libpopt.so.0 /lib/i386-linux-gnu/libacl.so.1 /lib/i386-linux-gnu/libattr.so.1 && - tar cCfhz / /work/.ci/sd4/bs/bin/deps.tar.gz /usr/bin/rsync /lib/i386-linux-gnu/libpopt.so.0 - ' - touch .ci/sd4/bs/bin/rsync.deps - section_end get-opam-prereqs-in-docker - fi - fi - - # Bundle Opam prerequisites (ManyLinux or Linux Docker) - if [ -n "${docker_runner:-}" ] || [ -n "${dockcross_image:-}" ]; then - # Bundle for consumers of setup-dkml.yml - do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/bs/bin/rsync - fi - } - # Get Opam Cache do_get_opam_cache() { if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi @@ -4839,7 +4799,7 @@ variables: termargs=-it fi - exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" + exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross ${dockcross_cli_image_args} --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" EOF chmod +x .ci/sd4/run-with-env @@ -4851,7 +4811,7 @@ variables: echo '___________________' >&2 do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/run-with-env .ci/sd4/run-in-docker .ci/sd4/edr - elif [ -n "${docker_runner:-}" ]; then + elif [ "${in_docker:-}" = "true" ] && [ -n "${docker_runner:-}" ]; then cat >.ci/sd4/run-with-env <&2 + echo 'Setup DkML compiler on a desktop PC.' >&2 echo 'usage: setup-dkml-darwin_x86_64.sh [options]' >&2 echo 'Options:' >&2 @@ -227,7 +229,6 @@ usage() { echo " --PIN_DKML_BASE_COMPILER=. Defaults to: ${PIN_DKML_BASE_COMPILER}" >&2 echo " --PIN_DKML_BUILD_DESKTOP=. Defaults to: ${PIN_DKML_BUILD_DESKTOP}" >&2 echo " --PIN_DKML_C_PROBE=. Defaults to: ${PIN_DKML_C_PROBE}" >&2 - echo " --PIN_DKML_COMPILER_ENV=. Defaults to: ${PIN_DKML_COMPILER_ENV}" >&2 echo " --PIN_DKML_COMPILER_SRC=. Defaults to: ${PIN_DKML_COMPILER_SRC}" >&2 echo " --PIN_DKML_COMPONENT_XX_CONSOLE=. Defaults to: ${PIN_DKML_COMPONENT_XX_CONSOLE}" >&2 echo " --PIN_DKML_EXE_LIB=. Defaults to: ${PIN_DKML_EXE_LIB}" >&2 @@ -440,8 +441,6 @@ while getopts :h-: option; do PIN_DKML_BUILD_DESKTOP=*) PIN_DKML_BUILD_DESKTOP=${OPTARG#*=} ;; PIN_DKML_C_PROBE) fail "Option \"$OPTARG\" missing argument" ;; PIN_DKML_C_PROBE=*) PIN_DKML_C_PROBE=${OPTARG#*=} ;; - PIN_DKML_COMPILER_ENV) fail "Option \"$OPTARG\" missing argument" ;; - PIN_DKML_COMPILER_ENV=*) PIN_DKML_COMPILER_ENV=${OPTARG#*=} ;; PIN_DKML_COMPILER_SRC) fail "Option \"$OPTARG\" missing argument" ;; PIN_DKML_COMPILER_SRC=*) PIN_DKML_COMPILER_SRC=${OPTARG#*=} ;; PIN_DKML_COMPONENT_XX_CONSOLE) fail "Option \"$OPTARG\" missing argument" ;; @@ -840,7 +839,7 @@ cat > .ci/sd4/run-checkout-code.sh <<'end_of_script' # dockcross (used by Linux) since a Docker-in-Docker container can have # difficulties doing a git checkout (the Git credentials for any private # repositories are likely not present). We don't care about any private -# repositories for DKML but any code that extends this (ex. DKSDK) may +# repositories for DkML but any code that extends this (ex. DKSDK) may # need to use private repositories. set -euf @@ -993,6 +992,33 @@ fi # ------------------------------------------------------------------- +docker_fqin_preusername= # fully qualified image name (hostname[:port]/username/reponame[:tag]), the parts before the username (hostname[:port]/) +if [ -n "${docker_registry:-}" ]; then + docker_fqin_preusername="$docker_registry/" +fi + +# Extend dockcross. https://github.com/dockcross/dockcross#how-to-extend-dockcross-images +dockcross_image_id= +dockcross_cli_image_args= +if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then + echo "Doing docker build" + section_begin docker-build "Summary: docker build -t ${docker_fqin_preusername}dkml-workflows/dockcross" + + install -d .ci/sd4/docker-image + printf "FROM %s\nENV DEFAULT_DOCKCROSS_IMAGE %sdkml-workflows/dockcross:latest\nRUN if command -v apt-get; then apt-get install -y rsync %s && rm -rf /var/lib/apt/lists/*; fi\nRUN if command -v yum; then yum install -y rsync %s && yum clean all && rm -rf /var/cache/yum; fi" \ + "${dockcross_image:-}" "${docker_fqin_preusername}" "${dockcross_packages_apt:-}" "${dockcross_packages_yum:-}" >.ci/sd4/docker-image/Dockerfile + docker build -t "${docker_fqin_preusername}dkml-workflows/dockcross:latest" .ci/sd4/docker-image + + # Save image id to re-use for all remaining dockcross invocations + docker images --format "{{.ID}} {{.CreatedAt}}" | sort -rk 2 | awk 'NR==1{print $1}' | tee .ci/sd4/docker-image-id + dockcross_image_id=$(cat .ci/sd4/docker-image-id) + dockcross_cli_image_args="--image $dockcross_image_id" + + section_end docker-build +fi + +# ------------------------------------------------------------------- + section_begin setup-info "Summary: setup-dkml" SKIP_OPAM_MODIFICATIONS=${SKIP_OPAM_MODIFICATIONS:-false} # default is false @@ -1043,8 +1069,8 @@ original_opam_root=${original_opam_root} original_opam_root_cacheable=${original_opam_root_cacheable} unix_opam_root=${unix_opam_root} unix_opam_root_cacheable=${unix_opam_root_cacheable} +docker_registry=${docker_registry:-} dockcross_image=${dockcross_image:-} -dockcross_image_custom_prefix=${dockcross_image_custom_prefix:-} dockcross_run_extra_args=${dockcross_run_extra_args:-} docker_runner=${docker_runner:-} in_docker=${in_docker:-} @@ -1204,12 +1230,12 @@ install -d .ci/sd4/dist tar cf .ci/sd4/dist/run-with-env.tar -T /dev/null do_get_dockcross() { - if [ -n "${dockcross_image:-}" ]; then + if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then # The dockcross script is super-slow section_begin get-dockcross 'Get dockcross binary (ManyLinux)' install -d .ci/sd4 # shellcheck disable=SC2086 - docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_custom_prefix:-}${dockcross_image:-}" >.ci/sd4/dockcross.gen + docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_id}" >.ci/sd4/dockcross.gen # PROBLEM 1 # --------- @@ -1307,10 +1333,11 @@ if [ "\$BUILDER_UID" = 0 ] && [ "\$BUILDER_GID" = 0 ]; then --rm \ \${ARGS:-} \ -v "\$HOST_PWD":/work \ - ${dockcross_image_custom_prefix:-}${dockcross_image:-} ${dockcross_entrypoint} "\$@" + "${dockcross_image_id}" ${dockcross_entrypoint} "\$@" else HERE=\$(dirname "\$0") HERE=\$(cd "\$HERE" && pwd) + export OCI_EXE=docker # default to podman if available, which breaks complaining about HTTPS vs HTTP on GitHub Actions communicating to http (docker) local registry. exec "\$HERE/dockcross-real" "\$@" fi EOF @@ -1324,46 +1351,6 @@ EOF } do_get_dockcross -if [ -n "${dockcross_image:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync ]; then - section_begin get-opam-prereqs-in-dockcross 'Get Opam prerequisites (ManyLinux)' - install -d .ci/sd4/bs/bin - # Install rsync with 'yum' (ManyLinux) or 'apt' (dockcross/linux-x64, etc.) - # if not present. - # shellcheck disable=SC2016 - .ci/sd4/dockcross --args "${dockcross_run_extra_args:-}" sh -c 'if ! command -v rsync; then if command -v yum; then sudo yum install -y rsync; else sudo apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync; fi; fi && install $(command -v rsync) .ci/sd4/bs/bin' - section_end get-opam-prereqs-in-dockcross - fi -fi - -# Opam prerequisites for using opam (not for installing opam) - -{ - if [ -n "${docker_runner:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync.deps ]; then - section_begin get-opam-prereqs-in-docker 'Get Opam prerequisites (Linux Docker)' - install -d .ci/sd4/bs/bin - ${docker_runner} sh -c ' - apt-get update -qq -o=Dpkg::Use-Pty=0 && - apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync && - ldd /usr/bin/rsync && - ls -l /lib/i386-linux-gnu/libpopt.so.0 /lib/i386-linux-gnu/libacl.so.1 /lib/i386-linux-gnu/libattr.so.1 && - tar cCfhz / /work/.ci/sd4/bs/bin/deps.tar.gz /usr/bin/rsync /lib/i386-linux-gnu/libpopt.so.0 - ' - touch .ci/sd4/bs/bin/rsync.deps - section_end get-opam-prereqs-in-docker - fi - fi - - # Bundle Opam prerequisites (ManyLinux or Linux Docker) - if [ -n "${docker_runner:-}" ] || [ -n "${dockcross_image:-}" ]; then - # Bundle for consumers of setup-dkml.yml - do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/bs/bin/rsync - fi -} - # Get Opam Cache do_get_opam_cache() { if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi @@ -1538,7 +1525,7 @@ if [ "\$#" -ge 1 ] && [ "\$1" = "-it" ]; then termargs=-it fi -exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" +exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross ${dockcross_cli_image_args} --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" EOF chmod +x .ci/sd4/run-with-env @@ -1550,7 +1537,7 @@ EOF echo '___________________' >&2 do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/run-with-env .ci/sd4/run-in-docker .ci/sd4/edr - elif [ -n "${docker_runner:-}" ]; then + elif [ "${in_docker:-}" = "true" ] && [ -n "${docker_runner:-}" ]; then cat >.ci/sd4/run-with-env <&2 + echo 'Setup DkML compiler on a desktop PC.' >&2 echo 'usage: setup-dkml-linux_x86.sh [options]' >&2 echo 'Options:' >&2 @@ -234,7 +233,6 @@ usage() { echo " --PIN_DKML_BASE_COMPILER=. Defaults to: ${PIN_DKML_BASE_COMPILER}" >&2 echo " --PIN_DKML_BUILD_DESKTOP=. Defaults to: ${PIN_DKML_BUILD_DESKTOP}" >&2 echo " --PIN_DKML_C_PROBE=. Defaults to: ${PIN_DKML_C_PROBE}" >&2 - echo " --PIN_DKML_COMPILER_ENV=. Defaults to: ${PIN_DKML_COMPILER_ENV}" >&2 echo " --PIN_DKML_COMPILER_SRC=. Defaults to: ${PIN_DKML_COMPILER_SRC}" >&2 echo " --PIN_DKML_COMPONENT_XX_CONSOLE=. Defaults to: ${PIN_DKML_COMPONENT_XX_CONSOLE}" >&2 echo " --PIN_DKML_EXE_LIB=. Defaults to: ${PIN_DKML_EXE_LIB}" >&2 @@ -447,8 +445,6 @@ while getopts :h-: option; do PIN_DKML_BUILD_DESKTOP=*) PIN_DKML_BUILD_DESKTOP=${OPTARG#*=} ;; PIN_DKML_C_PROBE) fail "Option \"$OPTARG\" missing argument" ;; PIN_DKML_C_PROBE=*) PIN_DKML_C_PROBE=${OPTARG#*=} ;; - PIN_DKML_COMPILER_ENV) fail "Option \"$OPTARG\" missing argument" ;; - PIN_DKML_COMPILER_ENV=*) PIN_DKML_COMPILER_ENV=${OPTARG#*=} ;; PIN_DKML_COMPILER_SRC) fail "Option \"$OPTARG\" missing argument" ;; PIN_DKML_COMPILER_SRC=*) PIN_DKML_COMPILER_SRC=${OPTARG#*=} ;; PIN_DKML_COMPONENT_XX_CONSOLE) fail "Option \"$OPTARG\" missing argument" ;; @@ -847,7 +843,7 @@ cat > .ci/sd4/run-checkout-code.sh <<'end_of_script' # dockcross (used by Linux) since a Docker-in-Docker container can have # difficulties doing a git checkout (the Git credentials for any private # repositories are likely not present). We don't care about any private -# repositories for DKML but any code that extends this (ex. DKSDK) may +# repositories for DkML but any code that extends this (ex. DKSDK) may # need to use private repositories. set -euf @@ -1000,6 +996,33 @@ fi # ------------------------------------------------------------------- +docker_fqin_preusername= # fully qualified image name (hostname[:port]/username/reponame[:tag]), the parts before the username (hostname[:port]/) +if [ -n "${docker_registry:-}" ]; then + docker_fqin_preusername="$docker_registry/" +fi + +# Extend dockcross. https://github.com/dockcross/dockcross#how-to-extend-dockcross-images +dockcross_image_id= +dockcross_cli_image_args= +if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then + echo "Doing docker build" + section_begin docker-build "Summary: docker build -t ${docker_fqin_preusername}dkml-workflows/dockcross" + + install -d .ci/sd4/docker-image + printf "FROM %s\nENV DEFAULT_DOCKCROSS_IMAGE %sdkml-workflows/dockcross:latest\nRUN if command -v apt-get; then apt-get install -y rsync %s && rm -rf /var/lib/apt/lists/*; fi\nRUN if command -v yum; then yum install -y rsync %s && yum clean all && rm -rf /var/cache/yum; fi" \ + "${dockcross_image:-}" "${docker_fqin_preusername}" "${dockcross_packages_apt:-}" "${dockcross_packages_yum:-}" >.ci/sd4/docker-image/Dockerfile + docker build -t "${docker_fqin_preusername}dkml-workflows/dockcross:latest" .ci/sd4/docker-image + + # Save image id to re-use for all remaining dockcross invocations + docker images --format "{{.ID}} {{.CreatedAt}}" | sort -rk 2 | awk 'NR==1{print $1}' | tee .ci/sd4/docker-image-id + dockcross_image_id=$(cat .ci/sd4/docker-image-id) + dockcross_cli_image_args="--image $dockcross_image_id" + + section_end docker-build +fi + +# ------------------------------------------------------------------- + section_begin setup-info "Summary: setup-dkml" SKIP_OPAM_MODIFICATIONS=${SKIP_OPAM_MODIFICATIONS:-false} # default is false @@ -1050,8 +1073,8 @@ original_opam_root=${original_opam_root} original_opam_root_cacheable=${original_opam_root_cacheable} unix_opam_root=${unix_opam_root} unix_opam_root_cacheable=${unix_opam_root_cacheable} +docker_registry=${docker_registry:-} dockcross_image=${dockcross_image:-} -dockcross_image_custom_prefix=${dockcross_image_custom_prefix:-} dockcross_run_extra_args=${dockcross_run_extra_args:-} docker_runner=${docker_runner:-} in_docker=${in_docker:-} @@ -1211,12 +1234,12 @@ install -d .ci/sd4/dist tar cf .ci/sd4/dist/run-with-env.tar -T /dev/null do_get_dockcross() { - if [ -n "${dockcross_image:-}" ]; then + if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then # The dockcross script is super-slow section_begin get-dockcross 'Get dockcross binary (ManyLinux)' install -d .ci/sd4 # shellcheck disable=SC2086 - docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_custom_prefix:-}${dockcross_image:-}" >.ci/sd4/dockcross.gen + docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_id}" >.ci/sd4/dockcross.gen # PROBLEM 1 # --------- @@ -1314,10 +1337,11 @@ if [ "\$BUILDER_UID" = 0 ] && [ "\$BUILDER_GID" = 0 ]; then --rm \ \${ARGS:-} \ -v "\$HOST_PWD":/work \ - ${dockcross_image_custom_prefix:-}${dockcross_image:-} ${dockcross_entrypoint} "\$@" + "${dockcross_image_id}" ${dockcross_entrypoint} "\$@" else HERE=\$(dirname "\$0") HERE=\$(cd "\$HERE" && pwd) + export OCI_EXE=docker # default to podman if available, which breaks complaining about HTTPS vs HTTP on GitHub Actions communicating to http (docker) local registry. exec "\$HERE/dockcross-real" "\$@" fi EOF @@ -1331,46 +1355,6 @@ EOF } do_get_dockcross -if [ -n "${dockcross_image:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync ]; then - section_begin get-opam-prereqs-in-dockcross 'Get Opam prerequisites (ManyLinux)' - install -d .ci/sd4/bs/bin - # Install rsync with 'yum' (ManyLinux) or 'apt' (dockcross/linux-x64, etc.) - # if not present. - # shellcheck disable=SC2016 - .ci/sd4/dockcross --args "${dockcross_run_extra_args:-}" sh -c 'if ! command -v rsync; then if command -v yum; then sudo yum install -y rsync; else sudo apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync; fi; fi && install $(command -v rsync) .ci/sd4/bs/bin' - section_end get-opam-prereqs-in-dockcross - fi -fi - -# Opam prerequisites for using opam (not for installing opam) - -{ - if [ -n "${docker_runner:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync.deps ]; then - section_begin get-opam-prereqs-in-docker 'Get Opam prerequisites (Linux Docker)' - install -d .ci/sd4/bs/bin - ${docker_runner} sh -c ' - apt-get update -qq -o=Dpkg::Use-Pty=0 && - apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync && - ldd /usr/bin/rsync && - ls -l /lib/i386-linux-gnu/libpopt.so.0 /lib/i386-linux-gnu/libacl.so.1 /lib/i386-linux-gnu/libattr.so.1 && - tar cCfhz / /work/.ci/sd4/bs/bin/deps.tar.gz /usr/bin/rsync /lib/i386-linux-gnu/libpopt.so.0 - ' - touch .ci/sd4/bs/bin/rsync.deps - section_end get-opam-prereqs-in-docker - fi - fi - - # Bundle Opam prerequisites (ManyLinux or Linux Docker) - if [ -n "${docker_runner:-}" ] || [ -n "${dockcross_image:-}" ]; then - # Bundle for consumers of setup-dkml.yml - do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/bs/bin/rsync - fi -} - # Get Opam Cache do_get_opam_cache() { if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi @@ -1545,7 +1529,7 @@ if [ "\$#" -ge 1 ] && [ "\$1" = "-it" ]; then termargs=-it fi -exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" +exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross ${dockcross_cli_image_args} --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" EOF chmod +x .ci/sd4/run-with-env @@ -1557,7 +1541,7 @@ EOF echo '___________________' >&2 do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/run-with-env .ci/sd4/run-in-docker .ci/sd4/edr - elif [ -n "${docker_runner:-}" ]; then + elif [ "${in_docker:-}" = "true" ] && [ -n "${docker_runner:-}" ]; then cat >.ci/sd4/run-with-env <&2 + echo 'Setup DkML compiler on a desktop PC.' >&2 echo 'usage: setup-dkml-linux_x86_64.sh [options]' >&2 echo 'Options:' >&2 @@ -234,7 +233,6 @@ usage() { echo " --PIN_DKML_BASE_COMPILER=. Defaults to: ${PIN_DKML_BASE_COMPILER}" >&2 echo " --PIN_DKML_BUILD_DESKTOP=. Defaults to: ${PIN_DKML_BUILD_DESKTOP}" >&2 echo " --PIN_DKML_C_PROBE=. Defaults to: ${PIN_DKML_C_PROBE}" >&2 - echo " --PIN_DKML_COMPILER_ENV=. Defaults to: ${PIN_DKML_COMPILER_ENV}" >&2 echo " --PIN_DKML_COMPILER_SRC=. Defaults to: ${PIN_DKML_COMPILER_SRC}" >&2 echo " --PIN_DKML_COMPONENT_XX_CONSOLE=. Defaults to: ${PIN_DKML_COMPONENT_XX_CONSOLE}" >&2 echo " --PIN_DKML_EXE_LIB=. Defaults to: ${PIN_DKML_EXE_LIB}" >&2 @@ -447,8 +445,6 @@ while getopts :h-: option; do PIN_DKML_BUILD_DESKTOP=*) PIN_DKML_BUILD_DESKTOP=${OPTARG#*=} ;; PIN_DKML_C_PROBE) fail "Option \"$OPTARG\" missing argument" ;; PIN_DKML_C_PROBE=*) PIN_DKML_C_PROBE=${OPTARG#*=} ;; - PIN_DKML_COMPILER_ENV) fail "Option \"$OPTARG\" missing argument" ;; - PIN_DKML_COMPILER_ENV=*) PIN_DKML_COMPILER_ENV=${OPTARG#*=} ;; PIN_DKML_COMPILER_SRC) fail "Option \"$OPTARG\" missing argument" ;; PIN_DKML_COMPILER_SRC=*) PIN_DKML_COMPILER_SRC=${OPTARG#*=} ;; PIN_DKML_COMPONENT_XX_CONSOLE) fail "Option \"$OPTARG\" missing argument" ;; @@ -847,7 +843,7 @@ cat > .ci/sd4/run-checkout-code.sh <<'end_of_script' # dockcross (used by Linux) since a Docker-in-Docker container can have # difficulties doing a git checkout (the Git credentials for any private # repositories are likely not present). We don't care about any private -# repositories for DKML but any code that extends this (ex. DKSDK) may +# repositories for DkML but any code that extends this (ex. DKSDK) may # need to use private repositories. set -euf @@ -1000,6 +996,33 @@ fi # ------------------------------------------------------------------- +docker_fqin_preusername= # fully qualified image name (hostname[:port]/username/reponame[:tag]), the parts before the username (hostname[:port]/) +if [ -n "${docker_registry:-}" ]; then + docker_fqin_preusername="$docker_registry/" +fi + +# Extend dockcross. https://github.com/dockcross/dockcross#how-to-extend-dockcross-images +dockcross_image_id= +dockcross_cli_image_args= +if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then + echo "Doing docker build" + section_begin docker-build "Summary: docker build -t ${docker_fqin_preusername}dkml-workflows/dockcross" + + install -d .ci/sd4/docker-image + printf "FROM %s\nENV DEFAULT_DOCKCROSS_IMAGE %sdkml-workflows/dockcross:latest\nRUN if command -v apt-get; then apt-get install -y rsync %s && rm -rf /var/lib/apt/lists/*; fi\nRUN if command -v yum; then yum install -y rsync %s && yum clean all && rm -rf /var/cache/yum; fi" \ + "${dockcross_image:-}" "${docker_fqin_preusername}" "${dockcross_packages_apt:-}" "${dockcross_packages_yum:-}" >.ci/sd4/docker-image/Dockerfile + docker build -t "${docker_fqin_preusername}dkml-workflows/dockcross:latest" .ci/sd4/docker-image + + # Save image id to re-use for all remaining dockcross invocations + docker images --format "{{.ID}} {{.CreatedAt}}" | sort -rk 2 | awk 'NR==1{print $1}' | tee .ci/sd4/docker-image-id + dockcross_image_id=$(cat .ci/sd4/docker-image-id) + dockcross_cli_image_args="--image $dockcross_image_id" + + section_end docker-build +fi + +# ------------------------------------------------------------------- + section_begin setup-info "Summary: setup-dkml" SKIP_OPAM_MODIFICATIONS=${SKIP_OPAM_MODIFICATIONS:-false} # default is false @@ -1050,8 +1073,8 @@ original_opam_root=${original_opam_root} original_opam_root_cacheable=${original_opam_root_cacheable} unix_opam_root=${unix_opam_root} unix_opam_root_cacheable=${unix_opam_root_cacheable} +docker_registry=${docker_registry:-} dockcross_image=${dockcross_image:-} -dockcross_image_custom_prefix=${dockcross_image_custom_prefix:-} dockcross_run_extra_args=${dockcross_run_extra_args:-} docker_runner=${docker_runner:-} in_docker=${in_docker:-} @@ -1211,12 +1234,12 @@ install -d .ci/sd4/dist tar cf .ci/sd4/dist/run-with-env.tar -T /dev/null do_get_dockcross() { - if [ -n "${dockcross_image:-}" ]; then + if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then # The dockcross script is super-slow section_begin get-dockcross 'Get dockcross binary (ManyLinux)' install -d .ci/sd4 # shellcheck disable=SC2086 - docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_custom_prefix:-}${dockcross_image:-}" >.ci/sd4/dockcross.gen + docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_id}" >.ci/sd4/dockcross.gen # PROBLEM 1 # --------- @@ -1314,10 +1337,11 @@ if [ "\$BUILDER_UID" = 0 ] && [ "\$BUILDER_GID" = 0 ]; then --rm \ \${ARGS:-} \ -v "\$HOST_PWD":/work \ - ${dockcross_image_custom_prefix:-}${dockcross_image:-} ${dockcross_entrypoint} "\$@" + "${dockcross_image_id}" ${dockcross_entrypoint} "\$@" else HERE=\$(dirname "\$0") HERE=\$(cd "\$HERE" && pwd) + export OCI_EXE=docker # default to podman if available, which breaks complaining about HTTPS vs HTTP on GitHub Actions communicating to http (docker) local registry. exec "\$HERE/dockcross-real" "\$@" fi EOF @@ -1331,46 +1355,6 @@ EOF } do_get_dockcross -if [ -n "${dockcross_image:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync ]; then - section_begin get-opam-prereqs-in-dockcross 'Get Opam prerequisites (ManyLinux)' - install -d .ci/sd4/bs/bin - # Install rsync with 'yum' (ManyLinux) or 'apt' (dockcross/linux-x64, etc.) - # if not present. - # shellcheck disable=SC2016 - .ci/sd4/dockcross --args "${dockcross_run_extra_args:-}" sh -c 'if ! command -v rsync; then if command -v yum; then sudo yum install -y rsync; else sudo apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync; fi; fi && install $(command -v rsync) .ci/sd4/bs/bin' - section_end get-opam-prereqs-in-dockcross - fi -fi - -# Opam prerequisites for using opam (not for installing opam) - -{ - if [ -n "${docker_runner:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync.deps ]; then - section_begin get-opam-prereqs-in-docker 'Get Opam prerequisites (Linux Docker)' - install -d .ci/sd4/bs/bin - ${docker_runner} sh -c ' - apt-get update -qq -o=Dpkg::Use-Pty=0 && - apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync && - ldd /usr/bin/rsync && - ls -l /lib/i386-linux-gnu/libpopt.so.0 /lib/i386-linux-gnu/libacl.so.1 /lib/i386-linux-gnu/libattr.so.1 && - tar cCfhz / /work/.ci/sd4/bs/bin/deps.tar.gz /usr/bin/rsync /lib/i386-linux-gnu/libpopt.so.0 - ' - touch .ci/sd4/bs/bin/rsync.deps - section_end get-opam-prereqs-in-docker - fi - fi - - # Bundle Opam prerequisites (ManyLinux or Linux Docker) - if [ -n "${docker_runner:-}" ] || [ -n "${dockcross_image:-}" ]; then - # Bundle for consumers of setup-dkml.yml - do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/bs/bin/rsync - fi -} - # Get Opam Cache do_get_opam_cache() { if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi @@ -1545,7 +1529,7 @@ if [ "\$#" -ge 1 ] && [ "\$1" = "-it" ]; then termargs=-it fi -exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" +exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross ${dockcross_cli_image_args} --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" EOF chmod +x .ci/sd4/run-with-env @@ -1557,7 +1541,7 @@ EOF echo '___________________' >&2 do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/run-with-env .ci/sd4/run-in-docker .ci/sd4/edr - elif [ -n "${docker_runner:-}" ]; then + elif [ "${in_docker:-}" = "true" ] && [ -n "${docker_runner:-}" ]; then cat >.ci/sd4/run-with-env <.ci/sd4/docker-image/Dockerfile + docker build -t "${docker_fqin_preusername}dkml-workflows/dockcross:latest" .ci/sd4/docker-image + + # Save image id to re-use for all remaining dockcross invocations + docker images --format "{{.ID}} {{.CreatedAt}}" | sort -rk 2 | awk 'NR==1{print $1}' | tee .ci/sd4/docker-image-id + dockcross_image_id=$(cat .ci/sd4/docker-image-id) + dockcross_cli_image_args="--image $dockcross_image_id" + + section_end docker-build +fi + +# ------------------------------------------------------------------- + section_begin setup-info "Summary: setup-dkml" SKIP_OPAM_MODIFICATIONS=${SKIP_OPAM_MODIFICATIONS:-false} # default is false @@ -1272,8 +1294,8 @@ original_opam_root=${original_opam_root} original_opam_root_cacheable=${original_opam_root_cacheable} unix_opam_root=${unix_opam_root} unix_opam_root_cacheable=${unix_opam_root_cacheable} +docker_registry=${docker_registry:-} dockcross_image=${dockcross_image:-} -dockcross_image_custom_prefix=${dockcross_image_custom_prefix:-} dockcross_run_extra_args=${dockcross_run_extra_args:-} docker_runner=${docker_runner:-} in_docker=${in_docker:-} @@ -1433,12 +1455,12 @@ install -d .ci/sd4/dist tar cf .ci/sd4/dist/run-with-env.tar -T /dev/null do_get_dockcross() { - if [ -n "${dockcross_image:-}" ]; then + if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then # The dockcross script is super-slow section_begin get-dockcross 'Get dockcross binary (ManyLinux)' install -d .ci/sd4 # shellcheck disable=SC2086 - docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_custom_prefix:-}${dockcross_image:-}" >.ci/sd4/dockcross.gen + docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_id}" >.ci/sd4/dockcross.gen # PROBLEM 1 # --------- @@ -1536,10 +1558,11 @@ if [ "\$BUILDER_UID" = 0 ] && [ "\$BUILDER_GID" = 0 ]; then --rm \ \${ARGS:-} \ -v "\$HOST_PWD":/work \ - ${dockcross_image_custom_prefix:-}${dockcross_image:-} ${dockcross_entrypoint} "\$@" + "${dockcross_image_id}" ${dockcross_entrypoint} "\$@" else HERE=\$(dirname "\$0") HERE=\$(cd "\$HERE" && pwd) + export OCI_EXE=docker # default to podman if available, which breaks complaining about HTTPS vs HTTP on GitHub Actions communicating to http (docker) local registry. exec "\$HERE/dockcross-real" "\$@" fi EOF @@ -1553,46 +1576,6 @@ EOF } do_get_dockcross -if [ -n "${dockcross_image:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync ]; then - section_begin get-opam-prereqs-in-dockcross 'Get Opam prerequisites (ManyLinux)' - install -d .ci/sd4/bs/bin - # Install rsync with 'yum' (ManyLinux) or 'apt' (dockcross/linux-x64, etc.) - # if not present. - # shellcheck disable=SC2016 - .ci/sd4/dockcross --args "${dockcross_run_extra_args:-}" sh -c 'if ! command -v rsync; then if command -v yum; then sudo yum install -y rsync; else sudo apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync; fi; fi && install $(command -v rsync) .ci/sd4/bs/bin' - section_end get-opam-prereqs-in-dockcross - fi -fi - -# Opam prerequisites for using opam (not for installing opam) - -{ - if [ -n "${docker_runner:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync.deps ]; then - section_begin get-opam-prereqs-in-docker 'Get Opam prerequisites (Linux Docker)' - install -d .ci/sd4/bs/bin - ${docker_runner} sh -c ' - apt-get update -qq -o=Dpkg::Use-Pty=0 && - apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync && - ldd /usr/bin/rsync && - ls -l /lib/i386-linux-gnu/libpopt.so.0 /lib/i386-linux-gnu/libacl.so.1 /lib/i386-linux-gnu/libattr.so.1 && - tar cCfhz / /work/.ci/sd4/bs/bin/deps.tar.gz /usr/bin/rsync /lib/i386-linux-gnu/libpopt.so.0 - ' - touch .ci/sd4/bs/bin/rsync.deps - section_end get-opam-prereqs-in-docker - fi - fi - - # Bundle Opam prerequisites (ManyLinux or Linux Docker) - if [ -n "${docker_runner:-}" ] || [ -n "${dockcross_image:-}" ]; then - # Bundle for consumers of setup-dkml.yml - do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/bs/bin/rsync - fi -} - # Get Opam Cache do_get_opam_cache() { if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi @@ -1767,7 +1750,7 @@ if [ "\$#" -ge 1 ] && [ "\$1" = "-it" ]; then termargs=-it fi -exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" +exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross ${dockcross_cli_image_args} --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" EOF chmod +x .ci/sd4/run-with-env @@ -1779,7 +1762,7 @@ EOF echo '___________________' >&2 do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/run-with-env .ci/sd4/run-in-docker .ci/sd4/edr - elif [ -n "${docker_runner:-}" ]; then + elif [ "${in_docker:-}" = "true" ] && [ -n "${docker_runner:-}" ]; then cat >.ci/sd4/run-with-env <.ci/sd4/docker-image/Dockerfile + docker build -t "${docker_fqin_preusername}dkml-workflows/dockcross:latest" .ci/sd4/docker-image + + # Save image id to re-use for all remaining dockcross invocations + docker images --format "{{.ID}} {{.CreatedAt}}" | sort -rk 2 | awk 'NR==1{print $1}' | tee .ci/sd4/docker-image-id + dockcross_image_id=$(cat .ci/sd4/docker-image-id) + dockcross_cli_image_args="--image $dockcross_image_id" + + section_end docker-build +fi + +# ------------------------------------------------------------------- + section_begin setup-info "Summary: setup-dkml" SKIP_OPAM_MODIFICATIONS=${SKIP_OPAM_MODIFICATIONS:-false} # default is false @@ -1271,8 +1293,8 @@ original_opam_root=${original_opam_root} original_opam_root_cacheable=${original_opam_root_cacheable} unix_opam_root=${unix_opam_root} unix_opam_root_cacheable=${unix_opam_root_cacheable} +docker_registry=${docker_registry:-} dockcross_image=${dockcross_image:-} -dockcross_image_custom_prefix=${dockcross_image_custom_prefix:-} dockcross_run_extra_args=${dockcross_run_extra_args:-} docker_runner=${docker_runner:-} in_docker=${in_docker:-} @@ -1432,12 +1454,12 @@ install -d .ci/sd4/dist tar cf .ci/sd4/dist/run-with-env.tar -T /dev/null do_get_dockcross() { - if [ -n "${dockcross_image:-}" ]; then + if [ "${in_docker:-}" = "true" ] && [ -n "${dockcross_image:-}" ]; then # The dockcross script is super-slow section_begin get-dockcross 'Get dockcross binary (ManyLinux)' install -d .ci/sd4 # shellcheck disable=SC2086 - docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_custom_prefix:-}${dockcross_image:-}" >.ci/sd4/dockcross.gen + docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_id}" >.ci/sd4/dockcross.gen # PROBLEM 1 # --------- @@ -1535,10 +1557,11 @@ if [ "\$BUILDER_UID" = 0 ] && [ "\$BUILDER_GID" = 0 ]; then --rm \ \${ARGS:-} \ -v "\$HOST_PWD":/work \ - ${dockcross_image_custom_prefix:-}${dockcross_image:-} ${dockcross_entrypoint} "\$@" + "${dockcross_image_id}" ${dockcross_entrypoint} "\$@" else HERE=\$(dirname "\$0") HERE=\$(cd "\$HERE" && pwd) + export OCI_EXE=docker # default to podman if available, which breaks complaining about HTTPS vs HTTP on GitHub Actions communicating to http (docker) local registry. exec "\$HERE/dockcross-real" "\$@" fi EOF @@ -1552,46 +1575,6 @@ EOF } do_get_dockcross -if [ -n "${dockcross_image:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync ]; then - section_begin get-opam-prereqs-in-dockcross 'Get Opam prerequisites (ManyLinux)' - install -d .ci/sd4/bs/bin - # Install rsync with 'yum' (ManyLinux) or 'apt' (dockcross/linux-x64, etc.) - # if not present. - # shellcheck disable=SC2016 - .ci/sd4/dockcross --args "${dockcross_run_extra_args:-}" sh -c 'if ! command -v rsync; then if command -v yum; then sudo yum install -y rsync; else sudo apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync; fi; fi && install $(command -v rsync) .ci/sd4/bs/bin' - section_end get-opam-prereqs-in-dockcross - fi -fi - -# Opam prerequisites for using opam (not for installing opam) - -{ - if [ -n "${docker_runner:-}" ]; then - # rsync needs to be available, even after Docker container disappears - if [ ! -e .ci/sd4/bs/bin/rsync.deps ]; then - section_begin get-opam-prereqs-in-docker 'Get Opam prerequisites (Linux Docker)' - install -d .ci/sd4/bs/bin - ${docker_runner} sh -c ' - apt-get update -qq -o=Dpkg::Use-Pty=0 && - apt-get install -qq -o=Dpkg::Use-Pty=0 -y rsync && - ldd /usr/bin/rsync && - ls -l /lib/i386-linux-gnu/libpopt.so.0 /lib/i386-linux-gnu/libacl.so.1 /lib/i386-linux-gnu/libattr.so.1 && - tar cCfhz / /work/.ci/sd4/bs/bin/deps.tar.gz /usr/bin/rsync /lib/i386-linux-gnu/libpopt.so.0 - ' - touch .ci/sd4/bs/bin/rsync.deps - section_end get-opam-prereqs-in-docker - fi - fi - - # Bundle Opam prerequisites (ManyLinux or Linux Docker) - if [ -n "${docker_runner:-}" ] || [ -n "${dockcross_image:-}" ]; then - # Bundle for consumers of setup-dkml.yml - do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/bs/bin/rsync - fi -} - # Get Opam Cache do_get_opam_cache() { if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi @@ -1766,7 +1749,7 @@ if [ "\$#" -ge 1 ] && [ "\$1" = "-it" ]; then termargs=-it fi -exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" +exec bash "\${PROJECT_DIR}"/.ci/sd4/dockcross ${dockcross_cli_image_args} --args "\${termargs} -v \${PROJECT_DIR}/.ci/sd4/edr:/home/root ${dockcross_run_extra_args:-}" /work/.ci/sd4/run-in-docker "\$@" EOF chmod +x .ci/sd4/run-with-env @@ -1778,7 +1761,7 @@ EOF echo '___________________' >&2 do_tar_rf .ci/sd4/dist/run-with-env.tar .ci/sd4/run-with-env .ci/sd4/run-in-docker .ci/sd4/edr - elif [ -n "${docker_runner:-}" ]; then + elif [ "${in_docker:-}" = "true" ] && [ -n "${docker_runner:-}" ]; then cat >.ci/sd4/run-with-env <