Merge branch 'main' into main

.devcontainer/Dockerfile

@@ -1,7 +1,7 @@
 # To find available Node images, see https://mcr.microsoft.com/en-us/product/devcontainers/javascript-node/tags
 
 # [Choice] Node.js version
-ARG VARIANT="dev-20-bullseye"
+ARG VARIANT="dev-22-bullseye"
 FROM mcr.microsoft.com/devcontainers/javascript-node:${VARIANT}
 
 # [Optional] Uncomment this section to install additional OS packages.

.devcontainer/devcontainer.json

@@ -2,70 +2,67 @@
 // https://github.com/microsoft/vscode-dev-containers/tree/v0.177.0/containers/javascript-node
 // -
 {
-    "name": "docs.github.com",
-    "build": {
-        "dockerfile": "Dockerfile",
-        // Update 'VARIANT' to pick a Node version
-        "args": { "VARIANT": "20" }
-    },
+  "name": "docs.github.com",
+  "build": {
+    "dockerfile": "Dockerfile",
+    // Update 'VARIANT' to pick a Node version
+    "args": { "VARIANT": "22" }
+  },
 
-    // Install features. Type 'feature' in the VS Code command palette for a full list.
-    "features": {
-        "sshd": "latest"
-    },
+  // Install features. Type 'feature' in the VS Code command palette for a full list.
+  "features": {
+    "sshd": "latest"
+  },
 
-    "customizations": {
-        "vscode": {
-            // Set *default* container specific settings.json values on container create.
-            "settings": {
-                "terminal.integrated.shell.linux": "/bin/bash",
-                "cSpell.language": ",en"
-            },
-            // Visual Studio Code extensions which help authoring for docs.github.com.
-            "extensions": [
-                "dbaeumer.vscode-eslint",
-                "sissel.shopify-liquid",
-                "davidanson.vscode-markdownlint",
-                "bierner.markdown-preview-github-styles",
-                "streetsidesoftware.code-spell-checker",
-                "alistairchristie.open-reusables",
-                "AlistairChristie.version-identifier",
-                "peterbe.ghdocs-goer",
-                "GitHub.copilot",
-                "GitHub.copilot-chat"
-            ]
-        },
-        "codespaces": {
-            "repositories": {
-                // allow Codespaces to pull from separate repo when user has access
-                "github/docs-early-access": {
-                    "permissions": {
-                        "contents": "write"
-                    }
-                }
-            }
-        }
+  "customizations": {
+    "vscode": {
+      // Set *default* container specific settings.json values on container create.
+      "settings": {
+        "terminal.integrated.shell.linux": "/bin/bash",
+        "cSpell.language": ",en"
+      },
+      // Visual Studio Code extensions which help authoring for docs.github.com.
+      "extensions": [
+        "dbaeumer.vscode-eslint",
+        "sissel.shopify-liquid",
+        "davidanson.vscode-markdownlint",
+        "bierner.markdown-preview-github-styles",
+        "streetsidesoftware.code-spell-checker",
+        "alistairchristie.open-reusables",
+        "AlistairChristie.version-identifier",
+        "peterbe.ghdocs-goer",
+        "GitHub.copilot",
+        "GitHub.copilot-chat"
+      ]
     },
-
-    // Use 'forwardPorts' to make a list of ports inside the container available locally.
-    "forwardPorts": [4000],
-
-    "portsAttributes": {
-        "4000": {
-            "label": "Preview"
+    "codespaces": {
+      "repositories": {
+        // allow Codespaces to pull from separate repo when user has access
+        "github/docs-early-access": {
+          "permissions": {
+            "contents": "write"
+          }
         }
-    },
+      }
+    }
+  },
 
-    // Use 'postCreateCommand' to run commands after the container is created.
-    "postCreateCommand": "npm ci",
+  // Use 'forwardPorts' to make a list of ports inside the container available locally.
+  "forwardPorts": [4000],
 
-    // Use 'updateContentCommand' to run commands to be included in Codespace pre-builds
-    "updateContentCommand": "git clone https://github.com/github/rest-api-description.git",
+  "portsAttributes": {
+    "4000": {
+      "label": "Preview"
+    }
+  },
 
-    // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
-    "remoteUser": "node",
+  // Use 'postCreateCommand' to run commands after the container is created.
+  "postCreateCommand": "npm ci",
 
-    "hostRequirements": {
-        "memory": "8gb"
-    }
+  // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+  "remoteUser": "node",
+
+  "hostRequirements": {
+    "memory": "8gb"
+  }
 }

.env.example

@@ -0,0 +1,23 @@
+# This file is a template for what your untracked .env file might look like for local development.
+# Please copy this to a new .env file and fill in the values as needed.
+
+# Requires a running local Elasticsearch service. Can be started via Docker, see https://github.com/github/docs-engineering/blob/main/docs/elasticsearch/elasticsearch-locally.md
+# When this value is unset searches will be proxied to the production Elasticsearch endpoint
+ELASTICSEARCH_URL=http://localhost:9200
+
+# Set for sending events in local development. See https://github.com/github/docs-engineering/blob/main/docs/analytics/hydro-mock.md
+HYDRO_ENDPOINT=
+HYDRO_SECRET=
+
+# Localization variables
+# See https://github.com/github/docs-internal/tree/main/src/languages#working-with-translated-content-locally
+ENABLED_LANGUAGES=
+TRANSLATIONS_ROOT=
+
+# For running the src/search/scripts/scrape script
+# You may want a lower value depending on your CPU
+BUILD_RECORDS_MAX_CONCURRENT=100
+BUILD_RECORDS_MIN_TIME=
+
+# Set to true to enable the /fastly-cache-test route for debugging Fastly headers
+ENABLE_FASTLY_TESTING=

.github/branch_protection_settings/main.json

@@ -41,6 +41,7 @@
       "workflows",
       "lint-code",
       "secret-scanning",
+      "pagelist"
     ],
     "contexts_url": "https://api.github.com/repos/github/docs-internal/branches/main/protection/required_status_checks/contexts",
     "checks": [
@@ -83,7 +84,8 @@
       { "context": "products", "app_id": 15368 },
       { "context": "workflows", "app_id": 15368 },
       { "context": "lint-code", "app_id": 15368 },
-      { "context": "secret-scanning", "app_id": 15368 }
+      { "context": "secret-scanning", "app_id": 15368 },
+      { "context": "pagelist", "app_id": 15368 }
     ]
   },
   "restrictions": {

.github/workflows/alert-changed-branch-protections.yml

@@ -3,6 +3,8 @@ name: Alert Changed Branch Protections
 on:
   branch_protection_rule:
   workflow_dispatch:
+  schedule:
+    - cron: '20 16 * * 3' # Run every Wednesday at 16:30 UTC / 8:30 PST
 
 permissions:
   contents: read

.github/workflows/azure-preview-env-deploy-public.yml

@@ -37,7 +37,15 @@ jobs:
   build-and-deploy-azure-preview-public:
     name: Build and deploy Azure preview environment (public)
     runs-on: ubuntu-latest
-    if: github.repository == 'github/docs'
+    # Ensure this is actually a pull request and not a merge group
+    # If its a merge group, report success without doing anything
+    # See https://bit.ly/3qB9nZW > If a job in a workflow is skipped due to a conditional, it will report its status as "Success".
+    if: |
+      (
+        (github.event.pull_request.head.sha)
+         && (github.event.number || github.run_id)
+       )
+       && github.repository == 'github/docs'
     timeout-minutes: 15
     environment:
       name: preview-env-${{ github.event.number }}

.github/workflows/azure-prod-build-deploy.yml

@@ -34,23 +34,14 @@ jobs:
       RESOURCE_GROUP_NAME: docs-prod
       APP_SERVICE_NAME: ghdocs-prod
       SLOT_NAME: canary
+      ACR_TOKEN_NAME: acrToken
 
     steps:
       - name: 'Az CLI login'
         uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # pin @v2
         with:
           creds: ${{ secrets.PROD_AZURE_CREDENTIALS }}
 
-      - name: 'Docker login'
-        uses: azure/docker-login@15c4aadf093404726ab2ff205b2cdd33fa6d054c
-        with:
-          login-server: ${{ secrets.PROD_REGISTRY_SERVER }}
-          username: ${{ secrets.PROD_REGISTRY_USERNAME }}
-          password: ${{ secrets.PROD_REGISTRY_PASSWORD }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db
-
       - name: Check out repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
@@ -64,11 +55,29 @@ jobs:
           node-version-file: 'package.json'
           cache: npm
 
-      # Currently we only need this to run dependencies in
-      # src/workflows/check-canary-slots.js
+      # We need this to run a few scripts that were easier to write in JS/TS
       - name: Install dependencies
         run: npm install
 
+      # Create a temporary token for the Azure Container Registry
+      # and set it as a GitHub Actions environment variable
+      # Created token are viewable in the ACR resource UI
+      # under Repository permissions > Tokens
+      - name: 'Create Azure Container Registry Token'
+        env:
+          CONTAINER_REGISTRY_SERVER: ${{ secrets.PROD_REGISTRY_SERVER }}
+        run: npm run create-acr-token
+
+      - name: 'Docker login'
+        uses: azure/docker-login@15c4aadf093404726ab2ff205b2cdd33fa6d054c
+        with:
+          login-server: ${{ secrets.PROD_REGISTRY_SERVER }}
+          username: ${{ env.ACR_TOKEN_NAME }}
+          password: ${{ env.ACR_TOKEN_VALUE }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db
+
       - name: Clone docs-early-access
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:

.github/workflows/index-autocomplete-elasticsearch.yml → .github/workflows/index-autocomplete-search.yml

@@ -1,7 +1,7 @@
-name: Index autocomplete Elasticsearch
+name: Index autocomplete search in Elasticsearch
 
-# **What it does**: Indexes autocomplete data into Elasticsearch.
-# **Why we have it**: So we can power the API for autocomplete.
+# **What it does**: Indexes autocomplete data (general and AI search) into Elasticsearch.
+# **Why we have it**: So we can power the APIs for autocomplete.
 # **Who does it impact**: docs-engineering
 
 on:
@@ -10,7 +10,7 @@ on:
     - cron: '20 16 * * *' # Run every day at 16:20 UTC / 8:20 PST
   pull_request:
     paths:
-      - .github/workflows/index-autocomplete-elasticsearch.yml
+      - .github/workflows/index-autocomplete-search.yml
       - 'src/search/scripts/index/**'
       - 'package*.json'
 
@@ -40,10 +40,15 @@ jobs:
         if: ${{ github.event_name == 'pull_request' }}
         run: curl --fail --retry-connrefused --retry 5 -I http://localhost:9200
 
-      - name: Run indexing
+      - name: Run general auto-complete indexing
         env:
           ELASTICSEARCH_URL: ${{ github.event_name == 'pull_request' && 'http://localhost:9200' || secrets.ELASTICSEARCH_URL }}
-        run: npm run index -- autocomplete docs-internal-data
+        run: npm run index-general-autocomplete -- docs-internal-data
+
+      - name: Run AI search auto-complete indexing
+        env:
+          ELASTICSEARCH_URL: ${{ github.event_name == 'pull_request' && 'http://localhost:9200' || secrets.ELASTICSEARCH_URL }}
+        run: npm run index-ai-search-autocomplete -- docs-internal-data
 
       - uses: ./.github/actions/slack-alert
         if: ${{ failure() && github.event_name == 'schedule' }}

.github/workflows/sync-search-pr.yml → .github/workflows/index-general-search-pr.yml

@@ -1,6 +1,6 @@
-name: Sync search - PR
+name: Index general search in Elasticsearch on PR
 
-# **What it does**: This does what `sync-sarch-elasticsearch.yml` does but
+# **What it does**: This does what `index-general-search-elasticsearch.yml` does but
 #                   with a localhost Elasticsearch and only for English.
 # **Why we have it**: To test that the script works and the popular pages json is valid.
 # **Who does it impact**: Docs engineering
@@ -11,8 +11,8 @@ on:
     paths:
       - 'src/search/**'
       - 'package*.json'
-      # Ultimately, for debugging this workflow itself
-      - .github/workflows/sync-search-pr.yml
+      # For debugging this workflow
+      - .github/workflows/index-general-search-pr.yml
       # Make sure we run this if the composite action changes
       - .github/actions/setup-elasticsearch/action.yml
 
@@ -25,9 +25,6 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  # Yes, it's hardcoded but it makes all the steps look exactly the same
-  # as they do in `sync-search-elasticsearch.yml` where it uses
-  # that `${{ env.ELASTICSEARCH_URL }}`
   ELASTICSEARCH_URL: http://localhost:9200
   # Since we'll run in NDOE_ENV=production, we need to be explicit that
   # we don't want Hydro configured.
@@ -63,7 +60,7 @@ jobs:
         env:
           ENABLE_DEV_LOGGING: false
         run: |
-          npm run sync-search-server > /tmp/stdout.log 2> /tmp/stderr.log &
+          npm run general-search-scrape-server > /tmp/stdout.log 2> /tmp/stderr.log &
 
           # first sleep to give it a chance to start
           sleep 6
@@ -88,15 +85,13 @@ jobs:
           # let's just accept an empty string instead.
           THROW_ON_EMPTY: false
 
-          # The sync-search-index recognizes this env var if you don't
-          # use the `--docs-internal-data <PATH>` option.
           DOCS_INTERNAL_DATA: docs-internal-data
 
         run: |
           mkdir /tmp/records
-          npm run sync-search-indices -- /tmp/records \
+          npm run general-search-scrape -- /tmp/records \
             --language en \
-            --version dotcom
+            --version fpt
 
           ls -lh /tmp/records
 
@@ -106,9 +101,9 @@ jobs:
 
       - name: Index into Elasticsearch
         run: |
-          npm run index-elasticsearch -- /tmp/records \
+          npm run index-general-search -- /tmp/records \
             --language en \
-            --version dotcom
+            --version fpt
 
       - name: Check created indexes and aliases
         run: |