From 3477939b798b64f135a09f1b8b54a0034ede3246 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Wed, 6 Mar 2019 20:14:27 -0800 Subject: [PATCH 1/9] [CI] Update binutils for powerpc64 and powerpc64le Cargo powerpc64 and powerpc64le are seeing `SIGILL` crashes in openssl, which was found to be a linking problem, fixed by newer binutils. See For powerpc64 we're using crosstool-ng, which doesn't offer a newer binutils version, but we can just compile it separately. On powerpc64le we're already building binutils. Both are now updated to binutils 2.32. Closes rust-lang/cargo#6320 Closes rust-lang/rust#57345 Closes rust-lang/rustup.rs#1620 --- .../build-powerpc64-toolchain.sh | 19 +++++++++++++++++++ .../build-powerpc64le-toolchain.sh | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/ci/docker/dist-powerpc64-linux/build-powerpc64-toolchain.sh b/src/ci/docker/dist-powerpc64-linux/build-powerpc64-toolchain.sh index f7aa2cd326832..a4431cbc0a16d 100755 --- a/src/ci/docker/dist-powerpc64-linux/build-powerpc64-toolchain.sh +++ b/src/ci/docker/dist-powerpc64-linux/build-powerpc64-toolchain.sh @@ -3,9 +3,28 @@ set -ex source shared.sh +BINUTILS=2.32 +TARGET=powerpc64-unknown-linux-gnu +PREFIX=/x-tools/$TARGET +SYSROOT=$PREFIX/$TARGET/sysroot + mkdir build cd build cp ../powerpc64-linux-gnu.config .config hide_output ct-ng build cd .. rm -rf build + +chmod -R u+w $PREFIX + +# Next, download and build newer binutils. +mkdir binutils-$TARGET +pushd binutils-$TARGET +curl https://ftp.gnu.org/gnu/binutils/binutils-$BINUTILS.tar.bz2 | tar xjf - +mkdir binutils-build +cd binutils-build +hide_output ../binutils-$BINUTILS/configure --target=$TARGET --prefix=$PREFIX --with-sysroot=$SYSROOT +hide_output make -j10 +hide_output make install +popd +rm -rf binutils-$TARGET diff --git a/src/ci/docker/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh b/src/ci/docker/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh index a01803d9c8f45..f866a24287f9e 100755 --- a/src/ci/docker/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh +++ b/src/ci/docker/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh @@ -4,7 +4,7 @@ set -ex source shared.sh -BINUTILS=2.25.1 +BINUTILS=2.32 GCC=5.3.0 TARGET=powerpc64le-linux-gnu SYSROOT=/usr/local/$TARGET/sysroot From c843fe710bb7fb454d68505c8000c0a50bc2a729 Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Wed, 6 Mar 2019 20:33:09 -0800 Subject: [PATCH 2/9] Wrap a long configure line --- .../docker/dist-powerpc64-linux/build-powerpc64-toolchain.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ci/docker/dist-powerpc64-linux/build-powerpc64-toolchain.sh b/src/ci/docker/dist-powerpc64-linux/build-powerpc64-toolchain.sh index a4431cbc0a16d..fc53849a2ada4 100755 --- a/src/ci/docker/dist-powerpc64-linux/build-powerpc64-toolchain.sh +++ b/src/ci/docker/dist-powerpc64-linux/build-powerpc64-toolchain.sh @@ -23,7 +23,8 @@ pushd binutils-$TARGET curl https://ftp.gnu.org/gnu/binutils/binutils-$BINUTILS.tar.bz2 | tar xjf - mkdir binutils-build cd binutils-build -hide_output ../binutils-$BINUTILS/configure --target=$TARGET --prefix=$PREFIX --with-sysroot=$SYSROOT +hide_output ../binutils-$BINUTILS/configure --target=$TARGET \ + --prefix=$PREFIX --with-sysroot=$SYSROOT hide_output make -j10 hide_output make install popd From 94a6936a692ed5e6f6c6a64d622d21ea4083e051 Mon Sep 17 00:00:00 2001 From: kennytm Date: Sat, 9 Mar 2019 14:28:25 +0800 Subject: [PATCH 3/9] Track embedded-book in the toolstate --- src/ci/docker/x86_64-gnu-tools/checktools.sh | 1 + src/tools/publish_toolstate.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/ci/docker/x86_64-gnu-tools/checktools.sh b/src/ci/docker/x86_64-gnu-tools/checktools.sh index 3343716419ff4..97e6ee25ec7a0 100755 --- a/src/ci/docker/x86_64-gnu-tools/checktools.sh +++ b/src/ci/docker/x86_64-gnu-tools/checktools.sh @@ -78,6 +78,7 @@ status_check() { check_dispatch $1 beta clippy-driver src/tools/clippy # these tools are not required for beta to successfully branch check_dispatch $1 nightly miri src/tools/miri + check_dispatch $1 nightly embedded-book src/doc/embedded-book } # If this PR is intended to update one of these tools, do not let the build pass diff --git a/src/tools/publish_toolstate.py b/src/tools/publish_toolstate.py index fb6132a5358ef..430db48954826 100755 --- a/src/tools/publish_toolstate.py +++ b/src/tools/publish_toolstate.py @@ -12,7 +12,7 @@ except ImportError: import urllib.request as urllib2 -# List of people to ping when the status of a tool changed. +# List of people to ping when the status of a tool or a book changed. MAINTAINERS = { 'miri': '@oli-obk @RalfJung @eddyb', 'clippy-driver': '@Manishearth @llogiq @mcarton @oli-obk @phansch', @@ -22,6 +22,7 @@ 'nomicon': '@frewsxcv @Gankro', 'reference': '@steveklabnik @Havvy @matthewjasper @alercah', 'rust-by-example': '@steveklabnik @marioidival @projektir', + 'embedded-book': '', } REPOS = { @@ -33,6 +34,7 @@ 'nomicon': 'https://github.com/rust-lang-nursery/nomicon', 'reference': 'https://github.com/rust-lang-nursery/reference', 'rust-by-example': 'https://github.com/rust-lang/rust-by-example', + 'embedded-book': 'https://github.com/rust-embedded/book', } @@ -70,7 +72,7 @@ def issue( cc @{}, the PR reviewer, and @rust-lang/compiler -- nominating for prioritization. - ''').format(relevant_pr_number, tool, REPOS[tool], relevant_pr_user, pr_reviewer), + ''').format(relevant_pr_number, tool, REPOS.get(tool), relevant_pr_user, pr_reviewer), 'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number), 'assignees': assignees, 'labels': ['T-compiler', 'I-nominated'], @@ -137,7 +139,7 @@ def update_latest( if build_failed: try: issue( - tool, MAINTAINERS.get(tool), + tool, MAINTAINERS.get(tool, ''), relevant_pr_number, relevant_pr_user, pr_reviewer, ) except IOError as e: From 135b686db41f67c4241144e25277a90092b014f4 Mon Sep 17 00:00:00 2001 From: James Munns Date: Sun, 10 Mar 2019 17:38:16 +0800 Subject: [PATCH 4/9] Update src/tools/publish_toolstate.py Co-Authored-By: kennytm --- src/tools/publish_toolstate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/publish_toolstate.py b/src/tools/publish_toolstate.py index 430db48954826..6e06034db599a 100755 --- a/src/tools/publish_toolstate.py +++ b/src/tools/publish_toolstate.py @@ -22,7 +22,7 @@ 'nomicon': '@frewsxcv @Gankro', 'reference': '@steveklabnik @Havvy @matthewjasper @alercah', 'rust-by-example': '@steveklabnik @marioidival @projektir', - 'embedded-book': '', + 'embedded-book': '@adamgreig @andre-richter @jamesmunns @korken89 @ryankurte @thejpster @therealprof', } REPOS = { From d6f51006cd47ca3f1ccacc4e0faa345877653b81 Mon Sep 17 00:00:00 2001 From: kennytm Date: Sun, 10 Mar 2019 18:02:40 +0800 Subject: [PATCH 5/9] Fix tidy --- src/tools/publish_toolstate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/tools/publish_toolstate.py b/src/tools/publish_toolstate.py index 6e06034db599a..f2a585e627307 100755 --- a/src/tools/publish_toolstate.py +++ b/src/tools/publish_toolstate.py @@ -22,7 +22,10 @@ 'nomicon': '@frewsxcv @Gankro', 'reference': '@steveklabnik @Havvy @matthewjasper @alercah', 'rust-by-example': '@steveklabnik @marioidival @projektir', - 'embedded-book': '@adamgreig @andre-richter @jamesmunns @korken89 @ryankurte @thejpster @therealprof', + 'embedded-book': ( + '@adamgreig @andre-richter @jamesmunns @korken89 ' + '@ryankurte @thejpster @therealprof' + ), } REPOS = { From 2070414689be778caf1a1ca104c10a20053ba8ee Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Sat, 9 Mar 2019 15:48:58 -0800 Subject: [PATCH 6/9] CI: Set job names. This should make it easier to identify what each job is doing when looking at the Travis or Appveyor UI. - Set `name` for each job in Travis. - Move `CI_JOB_NAME` to the front in Appveyor so that it appears first in the UI. --- .travis.yml | 54 ++++++++++++++++++++++++++++++++++++++++++++------ appveyor.yml | 56 ++++++++++++++++++++++++++-------------------------- 2 files changed, 76 insertions(+), 34 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7985b6c0e191f..7a8772d7abd63 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,20 +12,27 @@ git: depth: 2 submodules: false +env: + global: + - CI_JOB_NAME=$TRAVIS_JOB_NAME + matrix: fast_finish: true include: # Images used in testing PR and try-build should be run first. - env: IMAGE=x86_64-gnu-llvm-6.0 RUST_BACKTRACE=1 + name: x86_64-gnu-llvm-6.0 if: type = pull_request OR branch = auto - env: IMAGE=dist-x86_64-linux DEPLOY=1 + name: dist-x86_64-linux if: branch = try OR branch = auto # "alternate" deployments, these are "nightlies" but have LLVM assertions # turned on, they're deployed to a different location primarily for # additional testing. - - env: IMAGE=dist-x86_64-linux DEPLOY_ALT=1 CI_JOB_NAME=dist-x86_64-linux-alt + - env: IMAGE=dist-x86_64-linux DEPLOY_ALT=1 + name: dist-x86_64-linux-alt if: branch = try OR branch = auto - env: > @@ -37,9 +44,9 @@ matrix: MACOSX_DEPLOYMENT_TARGET=10.7 NO_LLVM_ASSERTIONS=1 NO_DEBUG_ASSERTIONS=1 - CI_JOB_NAME=dist-x86_64-apple-alt os: osx osx_image: xcode9.3-moar + name: dist-x86_64-apple-alt if: branch = auto # macOS builders. These are placed near the beginning because they are very @@ -60,9 +67,9 @@ matrix: MACOSX_STD_DEPLOYMENT_TARGET=10.7 NO_LLVM_ASSERTIONS=1 NO_DEBUG_ASSERTIONS=1 - CI_JOB_NAME=x86_64-apple os: osx osx_image: xcode9.3-moar + name: x86_64-apple if: branch = auto - env: > @@ -74,9 +81,9 @@ matrix: MACOSX_STD_DEPLOYMENT_TARGET=10.7 NO_LLVM_ASSERTIONS=1 NO_DEBUG_ASSERTIONS=1 - CI_JOB_NAME=i686-apple os: osx osx_image: xcode9.3-moar + name: i686-apple if: branch = auto # OSX builders producing releases. These do not run the full test suite and @@ -95,9 +102,9 @@ matrix: NO_LLVM_ASSERTIONS=1 NO_DEBUG_ASSERTIONS=1 DIST_REQUIRE_ALL_TOOLS=1 - CI_JOB_NAME=dist-i686-apple os: osx osx_image: xcode9.3-moar + name: dist-i686-apple if: branch = auto - env: > @@ -110,81 +117,116 @@ matrix: NO_LLVM_ASSERTIONS=1 NO_DEBUG_ASSERTIONS=1 DIST_REQUIRE_ALL_TOOLS=1 - CI_JOB_NAME=dist-x86_64-apple os: osx osx_image: xcode9.3-moar + name: dist-x86_64-apple if: branch = auto # Linux builders, remaining docker images - env: IMAGE=arm-android + name: arm-android if: branch = auto - env: IMAGE=armhf-gnu + name: armhf-gnu if: branch = auto - env: IMAGE=dist-various-1 DEPLOY=1 + name: dist-various-1 if: branch = auto - env: IMAGE=dist-various-2 DEPLOY=1 + name: dist-various-2 if: branch = auto - env: IMAGE=dist-aarch64-linux DEPLOY=1 + name: dist-aarch64-linux if: branch = auto - env: IMAGE=dist-android DEPLOY=1 + name: dist-android if: branch = auto - env: IMAGE=dist-arm-linux DEPLOY=1 + name: dist-arm-linux if: branch = auto - env: IMAGE=dist-armhf-linux DEPLOY=1 + name: dist-armhf-linux if: branch = auto - env: IMAGE=dist-armv7-linux DEPLOY=1 + name: dist-armv7-linux if: branch = auto - env: IMAGE=dist-i586-gnu-i586-i686-musl DEPLOY=1 + name: dist-i586-gnu-i586-i686-musl if: branch = auto - env: IMAGE=dist-i686-freebsd DEPLOY=1 + name: dist-i686-freebsd if: branch = auto - env: IMAGE=dist-i686-linux DEPLOY=1 + name: dist-i686-linux if: branch = auto - env: IMAGE=dist-mips-linux DEPLOY=1 + name: dist-mips-linux if: branch = auto - env: IMAGE=dist-mips64-linux DEPLOY=1 + name: dist-mips64-linux if: branch = auto - env: IMAGE=dist-mips64el-linux DEPLOY=1 + name: dist-mips64el-linux if: branch = auto - env: IMAGE=dist-mipsel-linux DEPLOY=1 + name: dist-mipsel-linux if: branch = auto - env: IMAGE=dist-powerpc-linux DEPLOY=1 + name: dist-powerpc-linux if: branch = auto - env: IMAGE=dist-powerpc64-linux DEPLOY=1 + name: dist-powerpc64-linux if: branch = auto - env: IMAGE=dist-powerpc64le-linux DEPLOY=1 + name: dist-powerpc64le-linux if: branch = auto - env: IMAGE=dist-s390x-linux DEPLOY=1 + name: dist-s390x-linux if: branch = auto - env: IMAGE=dist-x86_64-freebsd DEPLOY=1 + name: dist-x86_64-freebsd if: branch = auto - env: IMAGE=dist-x86_64-musl DEPLOY=1 + name: dist-x86_64-musl if: branch = auto - env: IMAGE=dist-x86_64-netbsd DEPLOY=1 + name: dist-x86_64-netbsd if: branch = auto - env: IMAGE=asmjs + name: asmjs if: branch = auto - env: IMAGE=i686-gnu + name: i686-gnu if: branch = auto - env: IMAGE=i686-gnu-nopt + name: i686-gnu-nopt if: branch = auto - env: IMAGE=test-various + name: test-various if: branch = auto - env: IMAGE=x86_64-gnu + name: x86_64-gnu if: branch = auto - env: IMAGE=x86_64-gnu-full-bootstrap + name: x86_64-gnu-full-bootstrap if: branch = auto - env: IMAGE=x86_64-gnu-aux + name: x86_64-gnu-aux if: branch = auto - env: IMAGE=x86_64-gnu-tools + name: x86_64-gnu-tools if: branch = auto OR (type = pull_request AND commit_message =~ /(?i:^update.*\b(rls|rustfmt|clippy|miri|cargo)\b)/) - env: IMAGE=x86_64-gnu-debug + name: x86_64-gnu-debug if: branch = auto - env: IMAGE=x86_64-gnu-nopt + name: x86_64-gnu-nopt if: branch = auto - env: IMAGE=x86_64-gnu-distcheck + name: x86_64-gnu-distcheck if: branch = auto - env: IMAGE=mingw-check + name: mingw-check if: type = pull_request OR branch = auto - stage: publish toolstate diff --git a/appveyor.yml b/appveyor.yml index 2abf723de496e..ab0a29d2752b5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,34 +10,34 @@ environment: matrix: # 32/64 bit MSVC tests - - MSYS_BITS: 64 + - CI_JOB_NAME: x86_64-msvc + MSYS_BITS: 64 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler SCRIPT: python x.py test - CI_JOB_NAME: x86_64-msvc - - MSYS_BITS: 32 + - CI_JOB_NAME: i686-msvc-1 + MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc SCRIPT: make appveyor-subset-1 - CI_JOB_NAME: i686-msvc-1 - - MSYS_BITS: 32 + - CI_JOB_NAME: i686-msvc-2 + MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc SCRIPT: make appveyor-subset-2 - CI_JOB_NAME: i686-msvc-2 # MSVC aux tests - - MSYS_BITS: 64 + - CI_JOB_NAME: x86_64-msvc-aux + MSYS_BITS: 64 RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc - CI_JOB_NAME: x86_64-msvc-aux - - MSYS_BITS: 64 + - CI_JOB_NAME: x86_64-msvc-cargo + MSYS_BITS: 64 SCRIPT: python x.py test src/tools/cargotest src/tools/cargo RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc - CI_JOB_NAME: x86_64-msvc-cargo # MSVC tools tests - - MSYS_BITS: 64 + - CI_JOB_NAME: x86_64-msvc-tools + MSYS_BITS: 64 SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri - CI_JOB_NAME: x86_64-msvc-tools # 32/64-bit MinGW builds. # @@ -52,30 +52,31 @@ environment: # bucket, but they cleraly didn't originate there! The downloads originally # came from the mingw-w64 SourceForge download site. Unfortunately # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. - - MSYS_BITS: 32 + - CI_JOB_NAME: i686-mingw-1 + MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make appveyor-subset-1 MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 - CI_JOB_NAME: i686-mingw-1 - - MSYS_BITS: 32 + - CI_JOB_NAME: i686-mingw-2 + MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make appveyor-subset-2 MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 - CI_JOB_NAME: i686-mingw-2 - - MSYS_BITS: 64 + - CI_JOB_NAME: x86_64-mingw + MSYS_BITS: 64 SCRIPT: python x.py test RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z MINGW_DIR: mingw64 - CI_JOB_NAME: x86_64-mingw # 32/64 bit MSVC and GNU deployment - - RUST_CONFIGURE_ARGS: > + - CI_JOB_NAME: dist-x86_64-msvc + RUST_CONFIGURE_ARGS: > --build=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc --enable-full-tools @@ -83,8 +84,8 @@ environment: SCRIPT: python x.py dist DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 - CI_JOB_NAME: dist-x86_64-msvc - - RUST_CONFIGURE_ARGS: > + - CI_JOB_NAME: dist-i686-msvc + RUST_CONFIGURE_ARGS: > --build=i686-pc-windows-msvc --target=i586-pc-windows-msvc --enable-full-tools @@ -92,8 +93,8 @@ environment: SCRIPT: python x.py dist DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 - CI_JOB_NAME: dist-i686-msvc - - MSYS_BITS: 32 + - CI_JOB_NAME: dist-i686-mingw + MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools SCRIPT: python x.py dist MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror @@ -101,8 +102,8 @@ environment: MINGW_DIR: mingw32 DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 - CI_JOB_NAME: dist-i686-mingw - - MSYS_BITS: 64 + - CI_JOB_NAME: dist-x86_64-mingw + MSYS_BITS: 64 SCRIPT: python x.py dist RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror @@ -110,14 +111,13 @@ environment: MINGW_DIR: mingw64 DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 - CI_JOB_NAME: dist-x86_64-mingw # "alternate" deployment, see .travis.yml for more info - - MSYS_BITS: 64 + - CI_JOB_NAME: dist-x86_64-msvc-alt + MSYS_BITS: 64 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler SCRIPT: python x.py dist DEPLOY_ALT: 1 - CI_JOB_NAME: dist-x86_64-msvc-alt matrix: fast_finish: true From 5c0f902ef49b50d30a4275ce7b14398c74cb087a Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Sun, 17 Mar 2019 08:04:57 -0600 Subject: [PATCH 7/9] Update CI configuration for building Redox libraries --- src/ci/docker/disabled/dist-x86_64-redox/Dockerfile | 4 ++-- src/ci/docker/dist-various-1/install-x86_64-redox.sh | 10 ++-------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/ci/docker/disabled/dist-x86_64-redox/Dockerfile b/src/ci/docker/disabled/dist-x86_64-redox/Dockerfile index f4c25f791bc39..11a3acd68e3e8 100644 --- a/src/ci/docker/disabled/dist-x86_64-redox/Dockerfile +++ b/src/ci/docker/disabled/dist-x86_64-redox/Dockerfile @@ -7,8 +7,8 @@ COPY scripts/crosstool-ng.sh /scripts/ RUN sh /scripts/crosstool-ng.sh WORKDIR /tmp -COPY cross/install-x86_64-redox.sh /tmp/ -RUN ./install-x86_64-redox.sh +COPY dist-various-1/install-x86_64-redox.sh /scripts/ +RUN sh /scripts/install-x86_64-redox.sh COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh diff --git a/src/ci/docker/dist-various-1/install-x86_64-redox.sh b/src/ci/docker/dist-various-1/install-x86_64-redox.sh index c39be14941c90..67c2c633cf313 100755 --- a/src/ci/docker/dist-various-1/install-x86_64-redox.sh +++ b/src/ci/docker/dist-various-1/install-x86_64-redox.sh @@ -3,11 +3,5 @@ set -ex -apt-get update -apt-get install -y --no-install-recommends software-properties-common apt-transport-https - -apt-key adv --batch --yes --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys AA12E97F0881517F -add-apt-repository -y 'deb https://static.redox-os.org/toolchain/apt /' - -apt-get update -apt-get install -y x86-64-unknown-redox-gcc +wget -O - https://static.redox-os.org/toolchain/x86_64-unknown-redox/relibc-install.tar.gz | \ +tar --extract --gzip --directory /usr/local From 07aee1df4427bcc2be5e9f6be5c853a0f6987b40 Mon Sep 17 00:00:00 2001 From: kennytm Date: Sun, 17 Mar 2019 19:39:00 +0800 Subject: [PATCH 8/9] Calculate Docker cache hash precisely from Dockerfile's dependencies `src/ci/docker`, so that when files under `dist-x86_64-linux` is changed, its dependent image `dist-i686-linux` will also be rebuilt. However, this ultraconservative solution caused the `dist-i686-linux` to be rebuilt every time an irrelevant Dockerfile (e.g. the PowerPC ones) is changed, which increases the building time beyond 3 hours and forcing a spurious but expected failure. This commit instead parses the Dockerfile itself and look for the actual dependencies. The scripts needs to be copied into the Docker image, which must be done with the COPY command, so we just need to find all lines with a COPY command and add the source file into the hash calculator. Note: this script only handles single-lined COPY command in the form `COPY src1 src2 src3 dst`, since these are the only variant used inside this repository. --- src/ci/docker/run.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index b4426bbfb962c..ef151750af9bf 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -22,7 +22,18 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then hash_key=/tmp/.docker-hash-key.txt rm -f "${hash_key}" echo $image >> $hash_key - find $docker_dir -type f | sort | xargs cat >> $hash_key + + cat "$docker_dir/$image/Dockerfile" >> $hash_key + # Look for all source files involves in the COPY command + copied_files=/tmp/.docker-copied-files.txt + rm -f "$copied_files" + for i in $(sed -n -e 's/^COPY \(.*\) .*$/\1/p' "$docker_dir/$image/Dockerfile"); do + # List the file names + find "$docker_dir/$i" -type f >> $copied_files + done + # Sort the file names and cat the content into the hash key + sort $copied_files | xargs cat >> $hash_key + docker --version >> $hash_key cksum=$(sha512sum $hash_key | \ awk '{print $1}') From f8bffad4f1f0650ae56e17221d9443f9c2a6ab20 Mon Sep 17 00:00:00 2001 From: kennytm Date: Wed, 20 Mar 2019 04:30:49 +0800 Subject: [PATCH 9/9] Use `curl` instead of `wget` in install-x86_64-redox.sh --- src/ci/docker/dist-various-1/install-x86_64-redox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/docker/dist-various-1/install-x86_64-redox.sh b/src/ci/docker/dist-various-1/install-x86_64-redox.sh index 67c2c633cf313..339042bb6729a 100755 --- a/src/ci/docker/dist-various-1/install-x86_64-redox.sh +++ b/src/ci/docker/dist-various-1/install-x86_64-redox.sh @@ -3,5 +3,5 @@ set -ex -wget -O - https://static.redox-os.org/toolchain/x86_64-unknown-redox/relibc-install.tar.gz | \ +curl https://static.redox-os.org/toolchain/x86_64-unknown-redox/relibc-install.tar.gz | \ tar --extract --gzip --directory /usr/local