Add support for cross-repo triggers. (#22)

* add support for cross-repo triggers
EmbarkStudios · Mar 23, 2023 · 3fbd234 · 3fbd234
1 parent 13f5c4c
commit 3fbd234
Show file tree

Hide file tree

Showing 12 changed files with 620 additions and 250 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -34,33 +34,29 @@ steps:
     steps:
       - label: ":package: Validating packages"
         command: |
-          apt-get update && apt-get install -y swig
-          bash get-pants.sh
-          /root/bin/pants -ldebug package ::
+          . ci/init.sh yes
+          pants -ldebug package ::
 
         agents: *large
 
       - label: ":pants: Validating tailor"
         command: |
-          apt-get update && apt-get install -y swig
-          bash get-pants.sh
-          /root/bin/pants tailor --check ::
+          . ci/init.sh no
+          pants tailor --check ::
 
         agents: *small
 
       - label: ":python-black: :isort: Check-and-lint "
         command: |
-          apt-get update && apt-get install -y swig
-          bash get-pants.sh
-          /root/bin/pants update-build-files --check lint check ::
+          . ci/init.sh yes
+          pants update-build-files --check lint check ::
 
         agents: *small
 
       - label: ":pytest: Run tests"
         command: |
-          apt-get update && apt-get install -y swig
-          bash get-pants.sh
-          /root/bin/pants test ::
+          . ci/init.sh yes
+          pants test ::
 
         agents: *small
 
@@ -72,10 +68,19 @@ steps:
       - label: ":docker: Publishing docker packages"
         env:
           PANTS_CONCURRENT: true
+        if: build.branch == "main"
+        command: |
+          . ci/init.sh yes
+          pants run ci/publish-docker-images.py -- -v HEAD^
+
+        agents: *large
 
+      - label: ":docker: Publishing docker packages"
+        env:
+          PANTS_CONCURRENT: true
+        if: build.branch != "main"
         command: |
-          apt-get update && apt-get install -y swig
-          bash get-pants.sh
-          /root/bin/pants run ci/publish-docker-images.py -- -v origin/main
+          . ci/init.sh yes
+          pants run ci/publish-docker-images.py -- -v origin/main
 
         agents: *large
diff --git a/3rdparty/python/pytest.lock b/3rdparty/python/pytest.lock
@@ -6,7 +6,7 @@
 // {
 //   "version": 3,
 //   "valid_for_interpreter_constraints": [
-//     "CPython<3.10,>=3.9"
+//     "CPython==3.9.*"
 //   ],
 //   "generated_with_requirements": [
 //     "pytest-cov!=2.12.1,<3.1,>=2.12",
@@ -80,61 +80,61 @@
           "artifacts": [
             {
               "algorithm": "sha256",
-              "hash": "436313d129db7cf5b4ac355dd2bd3f7c7e5294af077b090b85de75f8458b8616",
-              "url": "https://files.pythonhosted.org/packages/9e/de/989111fbf70f00912513912b3111072680154ec713b3f3d523636832965d/coverage-7.2.1-pp37.pp38.pp39-none-any.whl"
+              "hash": "872d6ce1f5be73f05bea4df498c140b9e7ee5418bfa2cc8204e7f9b817caa968",
+              "url": "https://files.pythonhosted.org/packages/5e/92/cffba484a7a34ca08097f7991b72cbd85da4adc717c69c067bab2e1f612c/coverage-7.2.2-pp37.pp38.pp39-none-any.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "bdd3f2f285ddcf2e75174248b2406189261a79e7fedee2ceeadc76219b6faa0e",
-              "url": "https://files.pythonhosted.org/packages/1c/89/b2f00ae4d1267aeed0026e68b17a6357afe52c1f773dff9c595da4b88977/coverage-7.2.1-cp39-cp39-musllinux_1_1_i686.whl"
+              "hash": "9d22e94e6dc86de981b1b684b342bec5e331401599ce652900ec59db52940005",
+              "url": "https://files.pythonhosted.org/packages/06/ab/1dd1a592ca3dff0c0d1da895c609f6b8b6b3092a2cc06f5494826b7689c8/coverage-7.2.2-cp39-cp39-macosx_11_0_arm64.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "834c2172edff5a08d78e2f53cf5e7164aacabeb66b369f76e7bb367ca4e2d993",
-              "url": "https://files.pythonhosted.org/packages/2f/e3/15f85c7527577907ddf354f45ad212c3882012c92186d45baa553cff47e3/coverage-7.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"
+              "hash": "bc4803779f0e4b06a2361f666e76f5c2e3715e8e379889d02251ec911befd149",
+              "url": "https://files.pythonhosted.org/packages/30/0c/6e0a65466f8008b2001c0354782753a880617b15dba857eb7e97928313fc/coverage-7.2.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "f29351393eb05e6326f044a7b45ed8e38cb4dcc38570d12791f271399dc41431",
-              "url": "https://files.pythonhosted.org/packages/36/51/e0a33466be7083709db9325167349ffeeedc419bb73864f72c8280e25aae/coverage-7.2.1-cp39-cp39-musllinux_1_1_x86_64.whl"
+              "hash": "507e4720791977934bba016101579b8c500fb21c5fa3cd4cf256477331ddd988",
+              "url": "https://files.pythonhosted.org/packages/3c/12/51c5bdce0ad62a29bff429e99a9b2f23a0220cca89e768fc22694d4aeb4f/coverage-7.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "4dd34a935de268a133e4741827ae951283a28c0125ddcdbcbba41c4b98f2dfef",
-              "url": "https://files.pythonhosted.org/packages/44/39/5efa733d0508d31565faf2761998f2e0417bfbb5bf4269b64384c7f5031a/coverage-7.2.1-cp39-cp39-macosx_10_9_x86_64.whl"
+              "hash": "db8c2c5ace167fd25ab5dd732714c51d4633f58bac21fb0ff63b0349f62755a8",
+              "url": "https://files.pythonhosted.org/packages/47/21/632047c45edb08f0eb51f9e8e49591d23c15ff078b32e0a88ddf99f4e8c2/coverage-7.2.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "99f4dd81b2bb8fc67c3da68b1f5ee1650aca06faa585cbc6818dbf67893c6d58",
-              "url": "https://files.pythonhosted.org/packages/4a/80/44c734492221b7e065bd8a2856c3f2f4650664abad70aab5e22a9d71a984/coverage-7.2.1-cp39-cp39-musllinux_1_1_aarch64.whl"
+              "hash": "797aad79e7b6182cb49c08cc5d2f7aa7b2128133b0926060d0a8889ac43843be",
+              "url": "https://files.pythonhosted.org/packages/4a/73/031f7380e2912f78801d28120091ba2dcf5bc0ad689e958d6b970ce67cb7/coverage-7.2.2-cp39-cp39-musllinux_1_1_x86_64.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "e4d70c853f0546855f027890b77854508bdb4d6a81242a9d804482e667fff6e6",
-              "url": "https://files.pythonhosted.org/packages/7b/36/afdf9ae3cf36ad922599df32b5517f48be389761869907bf89670eecd362/coverage-7.2.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl"
+              "hash": "36dd42da34fe94ed98c39887b86db9d06777b1c8f860520e21126a75507024f2",
+              "url": "https://files.pythonhosted.org/packages/54/8a/db9d9cd24f96bb872eea151bb0d5c8cb6a96825b70a0cfaf07bceab2884d/coverage-7.2.2.tar.gz"
             },
             {
               "algorithm": "sha256",
-              "hash": "0f8318ed0f3c376cfad8d3520f496946977abde080439d6689d7799791457454",
-              "url": "https://files.pythonhosted.org/packages/7c/f6/3c1715c27e3251baa578b469f3c266b48623d3bce4ea08509a636e11447c/coverage-7.2.1-cp39-cp39-macosx_11_0_arm64.whl"
+              "hash": "bfe7085783cda55e53510482fa7b5efc761fad1abe4d653b32710eb548ebdd2d",
+              "url": "https://files.pythonhosted.org/packages/6f/e4/9576c24b37c941e70a4815cbbd44ff2779c83b821ab3a38022b6f530d61c/coverage-7.2.2-cp39-cp39-macosx_10_9_x86_64.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "c77f2a9093ccf329dd523a9b2b3c854c20d2a3d968b6def3b820272ca6732242",
-              "url": "https://files.pythonhosted.org/packages/8d/4a/3518606d4b110df4f3e77bd52c241ae8a84c6dc74fac7c2a8e809449e541/coverage-7.2.1.tar.gz"
+              "hash": "4f68ee32d7c4164f1e2c8797535a6d0a3733355f5861e0f667e37df2d4b07140",
+              "url": "https://files.pythonhosted.org/packages/70/74/7ceb96c38dac56e5894c3b0af6c4adfd0dd7f96521e8b829544d1dcec8fe/coverage-7.2.2-cp39-cp39-musllinux_1_1_aarch64.whl"
             },
             {
               "algorithm": "sha256",
-              "hash": "8a6450da4c7afc4534305b2b7d8650131e130610cea448ff240b6ab73d7eab63",
-              "url": "https://files.pythonhosted.org/packages/e5/a9/62aabc67971d2fd439474b05cfc25c852b28bb6dfe8082c5b665652899f5/coverage-7.2.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl"
+              "hash": "d52f0a114b6a58305b11a5cdecd42b2e7f1ec77eb20e2b33969d702feafdd016",
+              "url": "https://files.pythonhosted.org/packages/d9/55/f45a1d08ad1299c5199f3cf1baaa02fcffe347fdec8ddacc484858700ef6/coverage-7.2.2-cp39-cp39-musllinux_1_1_i686.whl"
             }
           ],
           "project_name": "coverage",
           "requires_dists": [
             "tomli; python_full_version <= \"3.11.0a6\" and extra == \"toml\""
           ],
           "requires_python": ">=3.7",
-          "version": "7.2.1"
+          "version": "7.2.2"
         },
         {
           "artifacts": [
@@ -385,7 +385,7 @@
     }
   ],
   "path_mappings": {},
-  "pex_version": "2.1.123",
+  "pex_version": "2.1.129",
   "pip_version": "20.3.4-patched",
   "prefer_older_binary": false,
   "requirements": [
@@ -395,7 +395,7 @@
     "pytest==7.0.1"
   ],
   "requires_python": [
-    "<3.10,>=3.9"
+    "==3.9.*"
   ],
   "resolver_version": "pip-2020-resolver",
   "style": "universal",

diff --git a/ci/emote-override.py b/ci/emote-override.py
@@ -0,0 +1,54 @@
+import logging
+import sys
+from dataclasses import dataclass
+
+import cliclasses
+import coloredlogs
+from cliclasses import Count
+
+from ci.util import pants
+
+
+@dataclass
+class Override:
+    """Overrides the Emote dependency in the requirements file"""
+
+    """The new commit to use"""
+    commit: str
+
+    """The logging verbosity"""
+    verbosity: Count = 0
+
+
+args = cliclasses.parse_command_line(Override)
+coloredlogs.install(level=max((2 - args.verbosity), 0) * 10)
+
+logging.info("-- Updating requirements file")
+
+with open("requirements.txt") as f:
+    lines = f.readlines()
+
+index = 0
+for idx, line in enumerate(lines):
+    if line.startswith("emote-rl"):
+        index = idx
+        break
+
+else:
+    logging.error("Failed to locate package to override in requirements.txt")
+    sys.exit(1)
+
+vcs_spec = f"git+https://github.com/EmbarkStudios/emote@{args.commit}#egg=emote-rl"
+new_line = f"emote-rl[atari,wandb,torch] @ {vcs_spec}"
+
+lines[index] = new_line
+
+logging.info("Setting line %s to '%s'", index, new_line)
+with open("requirements.txt", "w") as f:
+    f.writelines(lines)
+
+
+logging.info("-- Generating lockfiles")
+info = pants("generate-lockfiles", "::")
+
+sys.exit(0)
diff --git a/ci/init.sh b/ci/init.sh
@@ -0,0 +1,20 @@
+set -euxo pipefail
+if [ "$#" -lt 1 ] ; then
+	echo "usage: init.sh [yes|no]"
+	exit 1
+fi
+
+override_emote="$1"
+echo "--- Running init script: $override_emote"
+
+apt-get update && apt-get install -y swig
+bash get-pants.sh
+
+export PATH="/root/bin/:$PATH"
+
+if [[ "$override_emote" = "yes" && -v "EMOTE_COMMIT" ]]; then
+	echo "-- Overriding emote"
+	PANTS_CONCURRENT=true pants run ci/emote-override.py -- -v "$EMOTE_COMMIT"
+fi
+
+echo "--- Running main build step"
diff --git a/ci/publish-docker-images.py b/ci/publish-docker-images.py
@@ -1,6 +1,6 @@
 import json
 import logging
-import subprocess
+import os
 from dataclasses import dataclass
 from typing import List
 
@@ -9,32 +9,46 @@
 import empyc.dataclasses
 from cliclasses import Count
 
-VALID_TARGETS = {
-    "src/container:erupt-trainer",
-    "src/container:erupt-server",
-    "src/container:erupt-runner",
-}
+from ci.util import buildkite, pants
 
 
 @dataclass
 class Publisher:
     treeish: str
+    dry_run: bool
     verbosity: Count = 0
 
 
+@dataclass
+class ArtifactManifest:
+    """Data output from the publish command."""
+
+    exit_code: int
+    names: List[str]
+    published: bool
+    publisher: str
+    repository: str
+    status: str
+    target: str
+
+
+VALID_TARGETS = {
+    "src/container:erupt-trainer",
+    "src/container:erupt-server",
+    "src/container:erupt-runner",
+}
+
+
 args = cliclasses.parse_command_line(Publisher)
 coloredlogs.install(level=max((2 - args.verbosity), 0) * 10)
 
 logging.info("Fetching changed images since %s", args.treeish)
 
-info = subprocess.check_output(
-    [
-        "/root/bin/pants",
-        f"--changed-since={args.treeish}",
-        "--changed-dependees=transitive",
-        "--filter-target-type=oci_python_image",
-        "list",
-    ]
+info = pants(
+    f"--changed-since={args.treeish}",
+    "--changed-dependees=transitive",
+    "--filter-target-type=oci_python_image",
+    "list",
 )
 
 targets = []
@@ -50,45 +64,32 @@ class Publisher:
     logging.info("No work to do; exiting")
     exit(0)
 
-info = subprocess.check_output(
-    [
-        "/root/bin/pants",
-        "publish",
-        "--publish-output=artifacts.json",
-        *targets,
-    ]
-)
-
-
-@dataclass
-class ArtifactManifest:
-    """Data output from the publish command."""
+if args.dry_run:
+    logging.warning("Would have published:%s", "".join(f"\n\t- {t}" for t in targets))
+    manifests = []
 
-    exit_code: int
-    names: List[str]
-    published: bool
-    publisher: str
-    repository: str
-    status: str
-    target: str
+else:
+    pants("publish", "--publish-output=artifacts.json", *targets)
 
+    with open("artifacts.json", "r") as f_:
+        data = json.load(f_)
 
-with open("artifacts.json", "r") as f_:
-    data = json.load(f_)
+    manifests = [empyc.dataclasses.recursive_from_dict(ArtifactManifest, item) for item in data]
 
-manifests = [empyc.dataclasses.recursive_from_dict(ArtifactManifest, item) for item in data]
+metadata = {}
 
 for manifest in manifests:
     path = f'{manifest.repository.removesuffix(":latest")}@{manifest.names[0]}'
     logging.info("Published image %s", path)
 
     container = manifest.target.split("-")[-1]
-    subprocess.check_output(
-        [
-            "buildkite-agent",
-            "meta-data",
-            "set",
-            f"{container}_image_digest",
-            path,
-        ]
-    )
+    metadata[f"{container}_image_digest"] = path
+
+if "EMOTE_MESSAGE" in os.environ:
+    metadata.update({k: v for k, v in os.environ.items() if k.startswith("EMOTE") if v != ""})
+
+
+for k, v in metadata.items():
+    logging.info("Setting metadata: '%s' -> '%s'", k, v)
+    if not args.dry_run:
+        buildkite("meta-data", "set", k, v)