build: Try to fix repo tests unsuccessfully. (#78)

.github/workflows/build_gems.yml

@@ -37,7 +37,7 @@ jobs:
             bazel-repos-${{ runner.os }}-
             bazel-repos-
       - name: "⚙️ Setup Bazel"
-        run: VERSION="1993.16.5" .github/workflows/setup-bazel.sh
+        run: .github/workflows/setup-bazel.sh
       - name: "🔎 Identify OS"
         run: echo "OS=$(uname -s | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_ENV"
       - name: ${{ format('🏗 Build ({0})', matrix.config) }}

.github/workflows/ci.yml

@@ -54,4 +54,24 @@ jobs:
       #     ls /usr/bin /usr/local/bin /bin
       #     # Technically, this is for Ruby 2.7.0 but it's probably fine.
       #     sudo apt-get install ruby2.7-dev
-      #     ./bazel test //test/scip/repos --test_env GITHUB_ACTIONS=1 --test_env GEM_PATH="$(dirname "$(command -v gem)")" --config=dbg
+      #     ./bazel test //test/scip/repos --test_env GITHUB_ACTIONS=1 --test_env GEM_PATH="$(dirname "$(command -v gem)")" --config=dbg
+      #
+      # Blocked on a bundler bug in 2.3.4. I get a crash trace like this when running locally.
+      #
+      # NoMethodError: undefined method `metadata' for nil:NilClass
+      #   /Users/varun/Code/scip-ruby/.cache_ruby/versions/2.7.2/lib/ruby/gems/2.7.0/gems/bundler-2.3.4/lib/bundler/cli/common.rb:22:in `block in output_fund_metadata_summary'
+      #   /Users/varun/Code/scip-ruby/.cache_ruby/versions/2.7.2/lib/ruby/gems/2.7.0/gems/bundler-2.3.4/lib/bundler/cli/common.rb:22:in `count'
+      #   /Users/varun/Code/scip-ruby/.cache_ruby/versions/2.7.2/lib/ruby/gems/2.7.0/gems/bundler-2.3.4/lib/bundler/cli/common.rb:22:in `output_fund_metadata_summary'
+      #   /Users/varun/Code/scip-ruby/.cache_ruby/versions/2.7.2/lib/ruby/gems/2.7.0/gems/bundler-2.3.4/lib/bundler/cli/install.rb:87:in `run'
+      #
+      # This is the method in question:
+      #
+      # def self.output_fund_metadata_summary
+      #   definition = Bundler.definition
+      #   current_dependencies = definition.requested_dependencies
+      #   current_specs = definition.specs
+      #
+      #   count = current_dependencies.count {|dep| current_specs[dep.name].first.metadata.key?("funding_uri") }
+      #                                                                          ^ boom!
+      # We should probably file a bug against bundler but I haven't been able to repro
+      # the issue outside of a Bazel sandbox yet.

docs/scip-ruby/CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 This document covers the day-to-day aspects of `scip-ruby`.
 For questions on why certain things are the way they are,
-see the [Design Decisions doc][] doc.
+see the [Design Decisions doc][].
 
 [Design Decisions doc]: DESIGN.md
 
@@ -114,9 +114,8 @@ For more information about configurations, see the [Sorbet README](./sorbet-READ
 ```
 
 The generated gems are located in `bazel-bin/gems/scip-ruby`.
-The gems have a version of `0.0.0`.
-To use a different version, pass `--//gem/scip-ruby:version=M.N.P`.
-
+For testing in local builds and CI, the gems use a hard-coded version `1993.5.16`.
+To build a different version, pass `--//gem/scip-ruby:version=M.N.P`.
 ## IDE Integration
 
 Generate `compile_commands.json` as per the [Sorbet README](./sorbet-README.md),

docs/scip-ruby/DESIGN.md

@@ -4,6 +4,10 @@
 
 <!-- DEF NOTE[repo-test-structure] -->
 
+WARNING: Right now, the repo tests are broken/disabled,
+so this information is a bit inaccurate. We should fix
+this once the repo tests are fixed.
+
 The repo tests have a couple of unusual things:
 #. Tests are broken up into 3 stages,
    instead of being a simple script that does clone

gems/scip-ruby/BUILD

@@ -2,7 +2,9 @@ load(":build_gems.bzl", "build_gems", "setup_standalone_ruby", "version")
 
 version(
     name = "version",
-    build_setting_default = "0.0.0",
+    # Don't set this to 0.0.0, that crashes some versions of Bundler.
+    # See https://github.com/rubygems/rubygems/pull/5116
+    build_setting_default = "1993.5.16",
 )
 
 setup_standalone_ruby(

gems/scip-ruby/build_gems.bzl

@@ -18,6 +18,7 @@ def _setup_standalone_ruby(ctx):
     runfiles = ctx.runfiles(files = outputs)
     return [DefaultInfo(files = depset(outputs), runfiles = runfiles)]
 
+# Not a genrule because we do not want PATH to propagate by default.
 setup_standalone_ruby = rule(
     implementation = _setup_standalone_ruby,
     attrs = {

gems/scip-ruby/install_standalone_ruby.sh

@@ -34,6 +34,17 @@ if [[ "$SCIP_RUBY_RBENV_EXE" != "RUNNING_IN_CI_RBENV_NOT_NEEDED" ]]; then
   RBENV_ROOT="$SCIP_RUBY_CACHE_RUBY_DIR" "$SCIP_RUBY_RBENV_EXE" install --skip-existing
 fi
 
+compress_deterministic() {
+  if [[ "$(tar --version)" == *"GNU"* ]]; then
+    tar -czf - -C "$1" . --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1993-05-16'
+  else
+    (cd "$1" && find . -print0 \
+      | sort -z \
+      | tar -czf - -T - --no-recursion --null --options='!timestamp')
+  fi
+}
+
 mkdir -p "$(dirname "$OUT_TGZ_PATH")"
 rm -f "$OUT_TGZ_PATH"
-tar -czf "$OUT_TGZ_PATH" -C "$SCIP_RUBY_CACHE_RUBY_DIR" .
+
+compress_deterministic "$SCIP_RUBY_CACHE_RUBY_DIR" > "$OUT_TGZ_PATH"

test/scip/repos/bundle_cache.sh

@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-ENV_VARS=("SCIP_RUBY_CACHE_RUBY_DIR" "RUBY_VERSION_FILE" "PRISTINE_TOOLCHAIN_TGZ_PATH" "TEST_GEM_ZIP" "TEST_GEM_ZIP_PREFIX" "TEST_GEM_SUBDIR" "OUT_MODIFIED_TOOLCHAIN_TGZ_PATH" "OUT_GEM_WITH_VENDOR_TGZ_PATH")
+ENV_VARS=("SCIP_RUBY_CACHE_RUBY_DIR" "RUBY_VERSION_FILE" "PRISTINE_TOOLCHAIN_TGZ_PATH" "OUT_MODIFIED_TOOLCHAIN_TGZ_PATH")
 for ENV_VAR in "${ENV_VARS[@]}"; do
   if eval "[ -z \"$(printf '${%s:-}' $ENV_VAR)\" ]"; then
     echo "Missing definition for $ENV_VAR environment variable"
@@ -25,42 +25,67 @@ tar -xzf "$PRISTINE_TOOLCHAIN_TGZ_PATH" -C "$SCIP_RUBY_CACHE_RUBY_DIR"
 GEM_EXE="$(find "$SCIP_RUBY_SPECIFIC_RUBY_ROOT" -name 'gem' -type f)"
 BUNDLE_EXE="$(dirname "$GEM_EXE")/bundle"
 
-rm -rf "$TEST_GEM_ZIP_PREFIX"
-unzip -q "$TEST_GEM_ZIP"
+compress_deterministic() {
+  if [[ "$(tar --version)" == *"GNU"* ]]; then
+    tar -czf - -C "$1" . --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1993-05-16'
+  else
+    (cd "$1" && find . -print0 \
+      | sort -z \
+      | tar -czf - -T - --no-recursion --null --options='!timestamp')
+  fi
+}
 
-# Is there a more reliable way to get the version? 😬
-_BUNDLER_VERSION="$(tail -n 1 < $TEST_GEM_ZIP_PREFIX/$TEST_GEM_SUBDIR/Gemfile.lock)"
-_BUNDLER_VERSION="${_BUNDLER_VERSION// /}"
-echo "BUNDLER_VERSION = $_BUNDLER_VERSION"
-"$GEM_EXE" install "bundler:$_BUNDLER_VERSION"
+bundle_cache_single_gem() {
+  rm -rf "$TEST_GEM_ZIP_PREFIX"
+  unzip -q "$TEST_GEM_ZIP"
 
-pushd "$TEST_GEM_ZIP_PREFIX/$TEST_GEM_SUBDIR"
-git init -q
-git add .
-GIT_AUTHOR_DATE=1993-05-16T15:04:05Z GIT_COMMITTER_DATE=1993-05-16T15:04:05Z \
-  git -c user.name='iu' -c user.email='i@u' commit -a -m '(^_^)' --quiet
-set +e
+  # Is there a more reliable way to get the version? 😬
+  _BUNDLER_VERSION="$(tail -n 1 < $TEST_GEM_ZIP_PREFIX/$TEST_GEM_SUBDIR/Gemfile.lock)"
+  _BUNDLER_VERSION="${_BUNDLER_VERSION// /}"
+  echo "BUNDLER_VERSION = $_BUNDLER_VERSION"
+  "$GEM_EXE" install "bundler:$_BUNDLER_VERSION"
 
-# For reasons that are unclear to me, the 'env -' usage works around
-# a gcc problem which manifests as 'cc1' not being executable when invoked
-# as 'gcc blah' and 'ld' not being found when invoked as '/usr/bin/gcc'.
-#
-# This seems specific to some Sorbet configuration; I have not been
-# able to reproduce this weirdness with a minimal configuration outside.
-if ! env - PATH="$PATH" PWD="$PWD" "$BUNDLE_EXE" cache --quiet 2> >(tee stderr.log >&2); then
-  err="$(< stderr.log)"
-  while IFS= read -r line; do
-    if [[ "$line" == *"lib/ruby/gems"*"mkmf.log" ]]; then
-      echo "----------- Printing likely bad mkmf.log -----------"
-      cat "$line"
-      echo "----------------------------------------------------"
-    fi
-  done <<< "$err"
-  exit 1
-fi
-set -e
-git restore .
-popd
+  pushd "$TEST_GEM_ZIP_PREFIX/$TEST_GEM_SUBDIR"
+  git init -q
+  git add .
+  GIT_AUTHOR_DATE=1993-05-16T15:04:05Z GIT_COMMITTER_DATE=1993-05-16T15:04:05Z \
+    git -c user.name='iu' -c user.email='i@u' commit -a -m '(^_^)' --quiet
+  set +e
+
+  # For reasons that are unclear to me, the 'env -' usage works around
+  # a gcc problem which manifests as 'cc1' not being executable when invoked
+  # as 'gcc blah' and 'ld' not being found when invoked as '/usr/bin/gcc'.
+  #
+  # This seems specific to some Sorbet configuration; I have not been
+  # able to reproduce this weirdness with a minimal configuration outside.
+  if ! env - PATH="$PATH" PWD="$PWD" "$BUNDLE_EXE" cache --quiet 2> >(tee stderr.log >&2); then
+    err="$(< stderr.log)"
+    while IFS= read -r line; do
+      if [[ "$line" == *"lib/ruby/gems"*"mkmf.log" ]]; then
+        echo "----------- Printing likely bad mkmf.log -----------"
+        cat "$line"
+        echo "----------------------------------------------------"
+      fi
+    done <<< "$err"
+    exit 1
+  fi
+  set -e
+  git restore .
+  popd
+  compress_deterministic "$TEST_GEM_ZIP_PREFIX" > "$OUT_GEM_WITH_VENDOR_TGZ_PATH"
+}
+
+for arg in "$@"; do
+  if [[ "$arg" =~ "zip="(.*)":prefix="(.*)":subdir="(.*)":out="(.*) ]]; then
+    TEST_GEM_ZIP="${BASH_REMATCH[1]}"
+    TEST_GEM_ZIP_PREFIX="${BASH_REMATCH[2]}"
+    TEST_GEM_SUBDIR="${BASH_REMATCH[3]}"
+    OUT_GEM_WITH_VENDOR_TGZ_PATH="${BASH_REMATCH[4]}"
+    bundle_cache_single_gem
+  else
+    echo 'error: Bazel-supplied arguments out-of-sync with expected pattern'
+    exit 1
+  fi
+done
 
-tar -czf "$OUT_MODIFIED_TOOLCHAIN_TGZ_PATH" -C "$SCIP_RUBY_CACHE_RUBY_DIR" .
-tar -czf "$OUT_GEM_WITH_VENDOR_TGZ_PATH" -C "$TEST_GEM_ZIP_PREFIX" .
+compress_deterministic "$SCIP_RUBY_CACHE_RUBY_DIR" > "$OUT_MODIFIED_TOOLCHAIN_TGZ_PATH"

test/scip/repos/index_gem.sh

@@ -39,15 +39,31 @@ if [ -n "${GITHUB_ACTIONS:-}" ]; then
 fi
 
 CACHED_TGZS_ARRAY=($CACHED_TGZS)
-TGZ_CACHE_DIR="$(dirname ${CACHED_TGZS_ARRAY[0]})"
-
-rm -rf "$SCIP_RUBY_CACHE_RUBY_DIR"
-mkdir -p "$SCIP_RUBY_CACHE_RUBY_DIR"
-tar -xzf "$TGZ_CACHE_DIR/ruby-post-bundle-cache.tgz" -C "$SCIP_RUBY_CACHE_RUBY_DIR"
+for CACHED_TGZ in "${CACHED_TGZS_ARRAY[@]}"; do
+  if [[ "$CACHED_TGZ" == *"/ruby-post-bundle-cache.tgz" ]]; then
+    rm -rf "$SCIP_RUBY_CACHE_RUBY_DIR"
+    mkdir -p "$SCIP_RUBY_CACHE_RUBY_DIR"
+    tar -xzf "$CACHED_TGZ" -C "$SCIP_RUBY_CACHE_RUBY_DIR"
+    FOUND_TOOLCHAIN_TGZ=1
+  else [[ "$CACHED_TGZ" == *"/$REPO_NAME/"* ]];
+    rm -rf repo
+    mkdir repo
+    tar -xzf "$CACHED_TGZ" -C repo
+    FOUND_REPO_TGZ=1
+  fi
+done
 
-rm -rf repo
-mkdir repo
-tar -xzf "$TGZ_CACHE_DIR/gem-post-vendor.tgz" -C repo
+if [[ -z "${FOUND_TOOLCHAIN_TGZ:-}" ]]; then
+  echo 'error: Did not find toolchain tgz in arguments; was it packaged?'
+  HAD_ERROR=1
+fi
+if [[ -z "${FOUND_REPO_TGZ:-}" ]]; then
+  echo 'error: Did not find toolchain tgz in arguments; was it packaged?'
+  HAD_ERROR=1
+fi
+if [[ -n "${HAD_ERROR:-}" ]]; then
+  exit 1
+fi
 
 SCIP_RUBY_GEMS_ARRAY=($SCIP_RUBY_GEMS)
 for SCIP_RUBY_GEM in "${SCIP_RUBY_GEMS_ARRAY[@]}"; do
@@ -73,8 +89,9 @@ pushd repo
 git diff --exit-code # No changes should've been made before applying the patch
 git apply "../$PATCH_PATH"
 
+"$BUNDLE_EXE" --version >&2
 "$BUNDLE_EXE" exec 'echo $PATH'
-"$BUNDLE_EXE" install -j "$(getconf _NPROCESSORS_ONLN)" --local
+"$BUNDLE_EXE" install --local
 
 set +e
 _INFO="$("$BUNDLE_EXE" info scip-ruby-debug)"
@@ -103,18 +120,14 @@ trim() {
 }
 GEM_INSTALL_PATH="$(trim "$GEM_INSTALL_PATH")"
 
-echo "GEM_INSTALL_PATH = $GEM_INSTALL_PATH"
-echo "Calling shim"
-
 {
-echo "Shim @ $GEM_INSTALL_PATH/bin/scip-ruby:"
-echo "-----------------------------------"
-cat "$GEM_INSTALL_PATH/bin/scip-ruby"
-echo "-----------------------------------"
-echo "This is for debugging/reproduction; I was seeing stale shims earlier, not sure why"
+  echo "Calling shim @ $GEM_INSTALL_PATH/bin/scip-ruby:"
+  echo "-----------------------------------"
+  cat "$GEM_INSTALL_PATH/bin/scip-ruby"
+  echo "-----------------------------------"
+  echo "This is for debugging/reproduction; I was seeing stale shims earlier, not sure why"
 } >&2
 
-
 # Call the shim, not the native binary, to test that the shim works.
 DEBUG=1 "$BUNDLE_EXE" exec "$GEM_INSTALL_PATH/bin/scip-ruby" --index-file index.scip --gem-metadata "$REPO_NAME@99.99.99"
 file index.scip