Dep signing (#8)

* add docker-image kind

* add docker dir

* back out to 9a2e2ec, fix one thing at a time

* back to b34543ca2f42771117fa9da33c31994913587636

* c78d65237c6ed064b9cbb3cd063b22e9e052a06a again

* node 14

* bad rebase

* address review comment

* really bad rebase

* add dep-signing kind back

* point back at the official repo master

.taskcluster.yml

@@ -9,12 +9,14 @@ tasks:
               branch: taskgraph
               revision: c78d65237c6ed064b9cbb3cd063b22e9e052a06a
           template:
-              repo: https://github.com/mozilla-extensions/xpi-template
+              repo: https://github.com/mozilla-releng/xpi-template
               branch: master
           trustDomain: xpi
           # XXX for private repos, uncomment this line, and comment out the next
           # githubCloneSecret: project/xpi/xpi-github-clone-ssh
           githubCloneSecret: ""
+          # XXX use "privileged" or "system" to enable signing
+          xpiSigningType: ""
       in:
           $if: 'tasks_for in ["github-pull-request", "github-push", "action", "cron"]'
           then:
@@ -179,6 +181,7 @@ tasks:
                                     XPI_HEAD_REF: '${head_branch}'
                                     XPI_HEAD_REV: '${head_sha}'
                                     XPI_REPOSITORY_TYPE: git
+                                    XPI_SIGNING_TYPE: '${xpiSigningType}'
                                     TEMPLATE_BASE_REPOSITORY: '${template.repo}'
                                     TEMPLATE_HEAD_REPOSITORY: '${template.repo}'
                                     TEMPLATE_HEAD_REV: '${template.branch}'

taskcluster/ci/build/kind.yml

@@ -17,7 +17,8 @@ job-defaults:
     worker:
         max-run-time: 7200
         docker-image:
-            indexed: xpi.cache.level-3.docker-images.v2.node-latest.latest
+            indexed: xpi.cache.level-3.docker-images.v2.node-14.latest
+        chain-of-trust: true
     run:
         checkout:
             xpi: {}

taskcluster/ci/config.yml

@@ -23,3 +23,11 @@ workers:
             implementation: docker-worker
             os: linux
             worker-type: 'b-linux'
+        dep-signing:
+            provisioner: scriptworker-k8s
+            implementation: scriptworker-signing
+            os: scriptworker
+            worker-type: xpi-t-signing
+
+scriptworker:
+    scope-prefix: project:xpi:releng

taskcluster/ci/dep-signing/kind.yml

@@ -0,0 +1,21 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+
+loader: xpi_taskgraph.single_dep:loader
+
+kind-dependencies:
+    - build
+
+transforms:
+    - xpi_taskgraph.signing:transforms
+    - taskgraph.transforms.task:transforms
+
+job-template:
+    description: Development-sign XPI
+    worker-type: dep-signing
+    worker:
+        signing-type: dep-signing
+        max-run-time: 3600
+    run-on-tasks-for: ['github-pull-request', 'github-push']

taskcluster/ci/test/kind.yml

@@ -17,7 +17,7 @@ job-defaults:
     worker:
         max-run-time: 7200
         docker-image:
-            indexed: xpi.cache.level-3.docker-images.v2.node-latest.latest
+            indexed: xpi.cache.level-3.docker-images.v2.node-14.latest
     run:
         checkout:
             xpi: {}

taskcluster/xpi_taskgraph/__init__.py

@@ -15,7 +15,9 @@ def register(graph_config):
     _import_modules([
         "build",
         "cached",
+        "single_dep",
         "test",
+        "worker_types",
         "xpi_manifest",
     ])
 

taskcluster/xpi_taskgraph/build.py

@@ -37,6 +37,8 @@ def tasks_from_manifest(config, jobs):
                 artifact_prefix = "xpi/build"
             else:
                 artifact_prefix = "public/build"
+            task.setdefault("attributes", {})
+            task["attributes"]["artifact-prefix"] = artifact_prefix
             env["ARTIFACT_PREFIX"] = artifact_prefix
             artifacts = task["worker"].setdefault("artifacts", [])
             artifacts.append(

taskcluster/xpi_taskgraph/signing.py

@@ -0,0 +1,84 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Apply some defaults and minor modifications to the jobs defined in the build
+kind.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.taskcluster import get_artifact_prefix
+from taskgraph.util.schema import resolve_keyed_by
+from taskgraph.util.keyed_by import evaluate_keyed_by
+
+
+transforms = TransformSequence()
+
+FORMATS = {
+    "privileged": "privileged_webextension",
+    "system": "system_addon",
+}
+
+
+@transforms.add
+def define_signing_flags(config, tasks):
+    for task in tasks:
+        dep = task["primary-dependency"]
+        # Current kind will be prepended later in the transform chain.
+        task["name"] = _get_dependent_job_name_without_its_kind(dep)
+        attributes = dep.attributes.copy()
+        if task.get("attributes"):
+            attributes.update(task["attributes"])
+        task["attributes"] = attributes
+        task["attributes"]["signed"] = True
+        if "run_on_tasks_for" in task["attributes"]:
+            task.setdefault("run-on-tasks-for", task["attributes"]["run_on_tasks_for"])
+
+        for key in ("worker-type", "worker.signing-type"):
+            resolve_keyed_by(
+                task,
+                key,
+                item_name=task["name"],
+                level=config.params["level"],
+            )
+        yield task
+
+
+@transforms.add
+def build_signing_task(config, tasks):
+    xpi_type = os.environ.get("XPI_SIGNING_TYPE", "unknown")
+    for task in tasks:
+        if xpi_type not in FORMATS:
+            continue
+        dep = task["primary-dependency"]
+        task["dependencies"] = {"build": dep.label}
+        artifact_prefix = get_artifact_prefix(dep.task)
+        if not artifact_prefix.startswith("public"):
+            scopes = task.setdefault('scopes', [])
+            scopes.append(
+                "queue:get-artifact:{}/*".format(dep.task["payload"]["env"]["ARTIFACT_PREFIX"].rstrip('/'))
+            )
+        xpi_name = dep.task["extra"]["xpi-name"]
+        # XXX until we can figure out what to sign, assume
+        #     `{artifact_prefix}/{xpi_name}.xpi`
+        path = "{}/{}.xpi".format(artifact_prefix, xpi_name)
+        format = FORMATS[xpi_type]
+        task["worker"]["upstream-artifacts"] = [
+            {
+                "taskId": {"task-reference": "<build>"},
+                "taskType": "build",
+                "paths": [path],
+                "formats": [format],
+            }
+        ]
+        task.setdefault("extra", {})["xpi-name"] = xpi_name
+        del task["primary-dependency"]
+        yield task
+
+
+def _get_dependent_job_name_without_its_kind(dependent_job):
+    return dependent_job.label[len(dependent_job.kind) + 1:]

taskcluster/xpi_taskgraph/single_dep.py

@@ -0,0 +1,46 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import copy
+
+from voluptuous import Required
+
+from taskgraph.task import Task
+from taskgraph.util.schema import Schema
+
+schema = Schema({Required("primary-dependency", "primary dependency task"): Task})
+
+
+def loader(kind, path, config, params, loaded_tasks):
+    """
+    Load tasks based on the jobs dependant kinds.
+
+    Optional `only-for-attributes` kind configuration, if specified, will limit
+    the jobs chosen to ones which have the specified attribute, with the specified
+    value.
+
+    Optional `job-template` kind configuration value, if specified, will be used to
+    pass configuration down to the specified transforms used.
+    """
+    only_attributes = config.get("only-for-attributes")
+    job_template = config.get("job-template")
+
+    for task in loaded_tasks:
+        if task.kind not in config.get("kind-dependencies", []):
+            continue
+
+        if only_attributes:
+            config_attrs = set(only_attributes)
+            if not config_attrs & set(task.attributes):
+                # make sure any attribute exists
+                continue
+
+        job = {"primary-dependency": task}
+
+        if job_template:
+            job.update(copy.deepcopy(job_template))
+
+        yield job

taskcluster/xpi_taskgraph/worker_types.py

@@ -0,0 +1,53 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from six import text_type
+
+from voluptuous import Required
+
+from taskgraph.util.schema import taskref_or_string
+from taskgraph.transforms.task import payload_builder
+
+
+@payload_builder(
+    "scriptworker-signing",
+    schema={
+        # the maximum time to run, in seconds
+        Required("max-run-time"): int,
+        Required("signing-type"): text_type,
+        # list of artifact URLs for the artifacts that should be signed
+        Required("upstream-artifacts"): [
+            {
+                # taskId of the task with the artifact
+                Required("taskId"): taskref_or_string,
+                # type of signing task (for CoT)
+                Required("taskType"): text_type,
+                # Paths to the artifacts to sign
+                Required("paths"): [text_type],
+                # Signing formats to use on each of the paths
+                Required("formats"): [text_type],
+            }
+        ],
+    },
+)
+def build_scriptworker_signing_payload(config, task, task_def):
+    worker = task["worker"]
+
+    task_def["tags"]["worker-implementation"] = "scriptworker"
+
+    task_def["payload"] = {
+        "maxRunTime": worker["max-run-time"],
+        "upstreamArtifacts": worker["upstream-artifacts"],
+    }
+
+    formats = set()
+    for artifacts in worker["upstream-artifacts"]:
+        formats.update(artifacts["formats"])
+
+    scope_prefix = config.graph_config["scriptworker"]["scope-prefix"]
+    task_def["scopes"].append(
+        "{}:signing:cert:{}".format(scope_prefix, worker["signing-type"])
+    )

taskcluster/xpi_taskgraph/xpi_manifest.py

@@ -46,7 +46,7 @@ def check_manifest(manifest_list):
     for k, v in xpi_names.items():
         if len(v) > 1:
             messages.append(
-                "Duplicate xpi name {} in directories {}\nTry renaming your subdirectories".format(
+                "Duplicate xpi name {} in directories {}\nTry renaming your xpis.".format(
                     k, v
                 )
             )
@@ -65,7 +65,7 @@ def get_manifest():
                 subdir_list.remove(dir_)
                 continue
         if "package.json" in file_list:
-            manifest = {"name": os.path.basename(dir_name), "tests": []}
+            manifest = {"tests": []}
             if "yarn.lock" in file_list:
                 manifest["install-type"] = "yarn"
             elif "package-lock.json" in file_list:
@@ -81,6 +81,7 @@ def get_manifest():
             for target in package_json.get("scripts", {}):
                 if target.startswith("test") or target.startswith("lint"):
                     manifest["tests"].append(target)
+            manifest["name"] = package_json["name"].lower()
             manifest_list.append(ReadOnlyDict(manifest))
     check_manifest(manifest_list[:])
     return tuple(manifest_list)