Merge branch 'cloudflare:main' into update-ai-gateway-run-types

.bazelrc

@@ -366,7 +366,7 @@ build:release_windows --copt="-fstrict-aliasing"
 build:windows --cxxopt='/std:c++20' --host_cxxopt='/std:c++20'
 build:windows --copt='/D_CRT_USE_BUILTIN_OFFSETOF' --host_copt='/D_CRT_USE_BUILTIN_OFFSETOF'
 build:windows --copt='/DWIN32_LEAN_AND_MEAN' --host_copt='/DWIN32_LEAN_AND_MEAN'
-# The `/std:c++20` argument is unused during boringssl compilation and we don't
+# The `/std:c++20` argument is unused during BoringSSL compilation and we don't
 # want a warning when compiling each file.
 build:windows --copt='-Wno-unused-command-line-argument' --host_copt='-Wno-unused-command-line-argument'
 

.github/workflows/internal-build.yml

@@ -0,0 +1,35 @@
+name: Run internal build
+
+on:
+  pull_request:
+    paths-ignore:
+      - 'doc/**'
+
+concurrency:
+  # Cancel existing builds for the same PR.
+  # Otherwise, all other builds will be allowed to run through.
+  group: internal-build.yml-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  internal-build:
+    if: "! ${{github.event.pull_request.head.repo.fork}}" # Don't run on forks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - name: Run internal build
+        env:
+          CI_URL: ${{ secrets.CI_URL }}
+          CI_CLIENT_ID: ${{ secrets.CI_CF_ACCESS_CLIENT_ID }}
+          CI_CLIENT_SECRET: ${{ secrets.CI_CF_ACCESS_CLIENT_SECRET }}
+        run: |
+          python3 ./tools/cross/internal_build.py \
+            ${{github.event.pull_request.number}} \
+            ${{github.event.pull_request.head.sha}} \
+            ${{github.run_attempt}} \
+            "${{github.event.pull_request.head.ref}}" \
+            $CI_URL \
+            $CI_CLIENT_ID \
+            $CI_CLIENT_SECRET

build/BUILD.wpt

@@ -2,19 +2,21 @@
 # Licensed under the Apache 2.0 license found in the LICENSE file or at:
 #     https://opensource.org/licenses/Apache-2.0
 
-directories = glob(
-    ["*"],
-    exclude = glob(
-        ["*"],
-        exclude_directories = 1,
-    ) + [
-        ".*",
+load("@workerd//:build/wpt_test.bzl", "wpt_get_directories")
+
+[filegroup(
+    name = dir,
+    srcs = glob(["{}/**/*".format(dir)]),
+    visibility = ["//visibility:public"],
+) for dir in wpt_get_directories(
+    excludes = [
+        "dom",
     ],
-    exclude_directories = 0,
-)
+    root = "",
+)]
 
 [filegroup(
     name = dir,
     srcs = glob(["{}/**/*".format(dir)]),
     visibility = ["//visibility:public"],
-) for dir in directories]
+) for dir in wpt_get_directories(root = "dom")]

build/deps/deps.jsonc

@@ -15,9 +15,9 @@
       "owner": "google",
       "repo": "boringssl",
       "branch": "master",
-      // boringssl may subtly break backwards compatibility and behave differently than the latest
+      // BoringSSL may subtly break backwards compatibility and behave differently than the latest
       // FIPS version, often by rejecting key values that it considers invalid/unsafe even though
-      // they are still accepted by boringssl. Update with caution and only after confirming this
+      // they are still accepted by BoringSSL. Update with caution and only after confirming this
       // is compatible with the downstream build.
       "freeze_commit": "6abe18402eb2a5e9b00158c6459646a948c53060"
     },

build/perfetto/perfetto_cfg.bzl

@@ -104,7 +104,7 @@ PERFETTO_CONFIG = struct(
     proto_library_visibility = "//visibility:private",
 
     # Allow Bazel embedders to change the visibility of the Go protos.
-    # Go protos have all sorts of strange behaviour in Google3 so need special
+    # Go protos have all sorts of strange behavior in Google3 so need special
     # handling as the rules for other languages do not work for Go.
     go_proto_library_visibility = "//visibility:private",
 

build/wd_ts_bundle.bzl

@@ -85,7 +85,7 @@ def wd_ts_bundle(
     )
 
     if lint:
-        # todo: lint js_srcs too, not just ts_srcs
+        # TODO: lint js_srcs too, not just ts_srcs
         eslint_bin.eslint_test(
             size = "large",
             name = name + "@eslint",

build/wpt_test.bzl

@@ -16,6 +16,8 @@ def wpt_test(name, wpt_directory, test_config):
 
     js_test_gen_rule = "{}@_wpt_js_test_gen".format(name)
     test_config_as_js = test_config.removesuffix(".ts") + ".js"
+    harness = "//src/wpt:wpt-test-harness"
+    compat_date = "//src/workerd/io:trimmed-supported-compatibility-date.txt"
 
     _wpt_js_test_gen(
         name = js_test_gen_rule,
@@ -31,22 +33,41 @@ def wpt_test(name, wpt_directory, test_config):
         wpt_directory = wpt_directory,
         test_config = test_config_as_js,
         test_js_generated = js_test_gen_rule,
+        harness = harness,
+        compat_date = compat_date,
     )
 
     wd_test(
         name = "{}".format(name),
         src = wd_test_gen_rule,
         args = ["--experimental"],
-        ts_deps = ["//src/wpt:wpt-test-harness"],
+        ts_deps = [harness],
         data = [
-            "//src/wpt:wpt-test-harness",
+            harness,
             test_config,
             js_test_gen_rule,
             wpt_directory,
-            "//src/workerd/io:trimmed-supported-compatibility-date.txt",
+            compat_date,
         ],
     )
 
+def wpt_get_directories(root, excludes = []):
+    """
+    Globs for files within a WPT directory structure, starting from root.
+    In addition to an explicitly provided excludes argument, hidden directories
+    and top-level files are also excluded as they don't contain test content.
+    """
+
+    root_pattern = "{}/*".format(root) if root else "*"
+    return native.glob(
+        [root_pattern],
+        exclude = native.glob(
+            [root_pattern],
+            exclude_directories = 1,
+        ) + [".*"] + excludes,
+        exclude_directories = 0,
+    )
+
 def _wpt_js_test_gen_impl(ctx):
     """
     Generates a workerd test suite in JS. This contains the logic to run
@@ -75,7 +96,7 @@ def generate_external_cases(files):
 
     for file in files.to_list():
         if file.extension == "js":
-            entry = """export const {} = run(config, '{}');""".format(test_case_name(file.basename), file.basename)
+            entry = """export const {} = run('{}');""".format(test_case_name(file.basename), file.basename)
             result.append(entry)
 
     return "\n".join(result)
@@ -97,9 +118,11 @@ def test_case_name(filename):
 
 WPT_JS_TEST_TEMPLATE = """// This file is autogenerated by wpt_test.bzl
 // DO NOT EDIT.
-import {{ run }} from 'wpt:harness';
+import {{ createRunner }} from 'wpt:harness';
 import config from '{test_config}';
 
+const run = createRunner(config);
+
 {cases}
 """
 
@@ -109,15 +132,16 @@ def _wpt_wd_test_gen_impl(ctx):
     paths to modules needed to run the test: generated test suite, test config
     file, WPT test scripts, associated JSON resources.
     """
-
     src = ctx.actions.declare_file("{}.wd-test".format(ctx.attr.test_name))
     ctx.actions.write(
         output = src,
         content = WPT_WD_TEST_TEMPLATE.format(
             test_name = ctx.attr.test_name,
             test_config = ctx.file.test_config.basename,
-            test_js_generated = wd_relative_path(ctx.file.test_js_generated),
-            bindings = generate_external_bindings(ctx.attr.wpt_directory.files),
+            test_js_generated = ctx.file.test_js_generated.basename,
+            bindings = generate_external_bindings(src.owner, ctx.attr.wpt_directory.files),
+            harness = wd_relative_path(src.owner, ctx.file.harness),
+            compat_date = wd_relative_path(src.owner, ctx.file.compat_date),
         ),
     )
 
@@ -134,14 +158,14 @@ const unitTests :Workerd.Config = (
         modules = [
           (name = "worker", esModule = embed "{test_js_generated}"),
           (name = "{test_config}", esModule = embed "{test_config}"),
-          (name = "wpt:harness", esModule = embed "../../../../../workerd/src/wpt/harness.js"),
+          (name = "wpt:harness", esModule = embed "{harness}"),
         ],
         bindings = [
           (name = "wpt", service = "wpt"),
           (name = "unsafe", unsafeEval = void),
           {bindings}
         ],
-        compatibilityDate = embed "../../../../../workerd/src/workerd/io/trimmed-supported-compatibility-date.txt",
+        compatibilityDate = embed "{compat_date}",
         compatibilityFlags = ["nodejs_compat", "experimental"],
       )
     ),
@@ -152,15 +176,14 @@ const unitTests :Workerd.Config = (
   ],
 );"""
 
-def wd_relative_path(file):
+def wd_relative_path(label, target):
     """
-    Returns a relative path which can be referenced in the .wd-test file.
-    This is four directories up from the bazel short_path
+    Generates a path that can be used in a .wd-test file to refer to another file. Paths are relative
+    to the .wd-test file. We determine the right path from the label that generated the .wd-test file
     """
+    return "../" * (label.package.count("/") + label.name.count("/") + 1) + target.short_path
 
-    return "../" * 4 + file.short_path
-
-def generate_external_bindings(files):
+def generate_external_bindings(label, files):
     """
     Generates appropriate bindings for each file in the WPT module:
     - JS files: text binding to allow code to be evaluated
@@ -170,7 +193,7 @@ def generate_external_bindings(files):
     result = []
 
     for file in files.to_list():
-        file_path = wd_relative_path(file)
+        file_path = wd_relative_path(label, file)
         if file.extension == "js":
             entry = """(name = "{}", text = embed "{}")""".format(file.basename, file_path)
         elif file.extension == "json":
@@ -195,6 +218,10 @@ _wpt_wd_test_gen = rule(
         "test_config": attr.label(allow_single_file = True),
         # An auto-generated JS file containing the test logic.
         "test_js_generated": attr.label(allow_single_file = True),
+        # Target specifying the location of the WPT test harness
+        "harness": attr.label(allow_single_file = True),
+        # Target specifying the location of the trimmed-supported-compatibility-date.txt file
+        "compat_date": attr.label(allow_single_file = True),
     },
 )
 

docs/benchmarking.md

@@ -5,7 +5,7 @@ We use a combination of micro and macro benchmarks for performance testing worke
 Benchmarks should be built using `--config=benchmark` configuration, which is builds a release
 binary with additional debug info.
 
-To obtain most consistent results it is recommended to disable cpu frequency scaling
+To obtain most consistent results it is recommended to disable CPU frequency scaling
 (use "performance" governor https://wiki.debian.org/CpuFrequencyScaling)
 
 # Micro benchmarks

samples/module_fallback/fallback.js

@@ -3,7 +3,7 @@
 const { createServer } = require('http');
 
 const server = createServer((req, res) => {
-  // The response from the fallback service must be a vaid JSON
+  // The response from the fallback service must be a valid JSON
   // serialization of a Worker::Module config.
 
   // The x-resolve-method tells us if the module was imported or required.