Merge branch 'main' into aibaars/limit-diagnostics

Author: aibaars

.github/workflows/build-ripunzip.yml

@@ -6,18 +6,18 @@ on:
       ripunzip-version:
         description: "what reference to checktout from google/runzip"
         required: false
-        default: v1.2.1
+        default: v2.0.2
       openssl-version:
         description: "what reference to checkout from openssl/openssl for Linux"
         required: false
-        default: openssl-3.3.0
+        default: openssl-3.5.0
 
 jobs:
   build:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, macos-13, windows-2019]
+        os: [ubuntu-22.04, macos-13, windows-2022]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4

.github/workflows/csharp-qltest.yml

@@ -36,7 +36,7 @@ jobs:
   unit-tests:
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-2019]
+        os: [ubuntu-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4

.github/workflows/codegen.yml renamed to .github/workflows/python-tooling.yml

@@ -1,10 +1,11 @@
-name: Codegen
+name: Python tooling
 
 on:
   pull_request:
     paths:
       - "misc/bazel/**"
       - "misc/codegen/**"
+      - "misc/scripts/models-as-data/bulk_generate_mad.py"
       - "*.bazel*"
       - .github/workflows/codegen.yml
       - .pre-commit-config.yaml
@@ -17,17 +18,17 @@ permissions:
   contents: read
 
 jobs:
-  codegen:
+  check-python-tooling:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version-file: 'misc/codegen/.python-version'
+          python-version: '3.12'
       - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507
         name: Check that python code is properly formatted
         with:
-          extra_args: autopep8 --all-files
+          extra_args: black --all-files
       - name: Run codegen tests
         shell: bash
         run: |

.github/workflows/validate-change-notes.yml

@@ -31,4 +31,4 @@ jobs:
       - name: Fail if there are any errors with existing change notes
 
         run: |
-          codeql pack release --groups cpp,csharp,java,javascript,python,ruby,-examples,-test,-experimental
+          codeql pack release --groups actions,cpp,csharp,go,java,javascript,python,ruby,shared,swift -examples,-test,-experimental

.pre-commit-config.yaml

@@ -1,5 +1,7 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
+default_language_version:
+  python: python3.12
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v3.2.0
@@ -14,11 +16,11 @@ repos:
     hooks:
       - id: clang-format
 
-  - repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v2.0.4
+  - repo: https://github.com/psf/black
+    rev: 25.1.0
     hooks:
-      - id: autopep8
-        files: ^misc/codegen/.*\.py
+      - id: black
+        files: ^(misc/codegen/.*|misc/scripts/models-as-data/bulk_generate_mad)\.py$
 
   - repo: local
     hooks:

CODEOWNERS

@@ -16,7 +16,7 @@
 /java/ql/test-kotlin2/ @github/codeql-kotlin
 
 # Experimental CodeQL cryptography
-**/experimental/quantum/ @github/ps-codeql
+**/experimental/**/quantum/ @github/ps-codeql
 /shared/quantum/ @github/ps-codeql
 
 # CodeQL tools and associated docs

MODULE.bazel

@@ -239,24 +239,24 @@ go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps")
 go_deps.from_file(go_mod = "//go/extractor:go.mod")
 use_repo(go_deps, "org_golang_x_mod", "org_golang_x_tools")
 
-lfs_files = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_files")
+lfs_archive = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_archive")
 
-lfs_files(
+lfs_archive(
     name = "ripunzip-linux",
-    srcs = ["//misc/ripunzip:ripunzip-linux"],
-    executable = True,
+    src = "//misc/ripunzip:ripunzip-Linux.zip",
+    build_file = "//misc/ripunzip:BUILD.ripunzip.bazel",
 )
 
-lfs_files(
+lfs_archive(
     name = "ripunzip-windows",
-    srcs = ["//misc/ripunzip:ripunzip-windows.exe"],
-    executable = True,
+    src = "//misc/ripunzip:ripunzip-Windows.zip",
+    build_file = "//misc/ripunzip:BUILD.ripunzip.bazel",
 )
 
-lfs_files(
+lfs_archive(
     name = "ripunzip-macos",
-    srcs = ["//misc/ripunzip:ripunzip-macos"],
-    executable = True,
+    src = "//misc/ripunzip:ripunzip-macOS.zip",
+    build_file = "//misc/ripunzip:BUILD.ripunzip.bazel",
 )
 
 register_toolchains(

actions/ql/lib/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.4.11
+
+No user-facing changes.
+
+## 0.4.10
+
+No user-facing changes.
+
 ## 0.4.9
 
 No user-facing changes.

actions/ql/lib/change-notes/2025-06-09-bash-parsing-performance.md

@@ -0,0 +1,6 @@
+---
+category: minorAnalysis
+---
+* Fixed performance issues in the parsing of Bash scripts in workflow files,
+  which led to out-of-disk errors when analysing certain workflow files with
+  complex interpolations of shell commands or quoted strings.

actions/ql/lib/change-notes/released/0.4.10.md

@@ -0,0 +1,3 @@
+## 0.4.10
+
+No user-facing changes.

actions/ql/lib/change-notes/released/0.4.11.md

@@ -0,0 +1,3 @@
+## 0.4.11
+
+No user-facing changes.

actions/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.4.9
+lastReleaseVersion: 0.4.11

actions/ql/lib/codeql/actions/Ast.qll

@@ -50,8 +50,8 @@ class Expression extends AstNode instanceof ExpressionImpl {
   string getNormalizedExpression() { result = normalizeExpr(expression) }
 }
 
-/** A common class for `env` in workflow, job or step. */
-abstract class Env extends AstNode instanceof EnvImpl {
+/** An `env` in workflow, job or step. */
+class Env extends AstNode instanceof EnvImpl {
   /** Gets an environment variable value given its name. */
   ScalarValueImpl getEnvVarValue(string name) { result = super.getEnvVarValue(name) }
 

actions/ql/lib/codeql/actions/Bash.qll

@@ -8,35 +8,64 @@ class BashShellScript extends ShellScript {
     )
   }
 
-  private string lineProducer(int i) {
-    result = this.getRawScript().regexpReplaceAll("\\\\\\s*\n", "").splitAt("\n", i)
-  }
-
-  private predicate cmdSubstitutionReplacement(string cmdSubs, string id, int k) {
-    exists(string line | line = this.lineProducer(k) |
-      exists(int i, int j |
-        cmdSubs =
-          // $() cmd substitution
-          line.regexpFind("\\$\\((?:[^()]+|\\((?:[^()]+|\\([^()]*\\))*\\))*\\)", i, j)
-              .regexpReplaceAll("^\\$\\(", "")
-              .regexpReplaceAll("\\)$", "") and
-        id = "cmdsubs:" + k + ":" + i + ":" + j
-      )
-      or
-      exists(int i, int j |
-        // `...` cmd substitution
-        cmdSubs =
-          line.regexpFind("\\`[^\\`]+\\`", i, j)
-              .regexpReplaceAll("^\\`", "")
-              .regexpReplaceAll("\\`$", "") and
-        id = "cmd:" + k + ":" + i + ":" + j
-      )
+  /**
+   * Gets the line at 0-based index `lineIndex` within this shell script,
+   * assuming newlines as separators.
+   */
+  private string lineProducer(int lineIndex) {
+    result = this.getRawScript().regexpReplaceAll("\\\\\\s*\n", "").splitAt("\n", lineIndex)
+  }
+
+  private predicate cmdSubstitutionReplacement(string command, string id, int lineIndex) {
+    this.commandInSubstitution(lineIndex, command, id)
+    or
+    this.commandInBackticks(lineIndex, command, id)
+  }
+
+  /**
+   * Holds if there is a command substitution `$(command)` in
+   * the line at `lineIndex` in the shell script,
+   * and `id` is a unique identifier for this command.
+   */
+  private predicate commandInSubstitution(int lineIndex, string command, string id) {
+    exists(int occurrenceIndex, int occurrenceOffset |
+      command =
+        // Look for the command inside a $(...) command substitution
+        this.lineProducer(lineIndex)
+            .regexpFind("\\$\\((?:[^()]+|\\((?:[^()]+|\\([^()]*\\))*\\))*\\)", occurrenceIndex,
+              occurrenceOffset)
+            // trim starting $( - TODO do this in first regex
+            .regexpReplaceAll("^\\$\\(", "")
+            // trim ending ) - TODO do this in first regex
+            .regexpReplaceAll("\\)$", "") and
+      id = "cmdsubs:" + lineIndex + ":" + occurrenceIndex + ":" + occurrenceOffset
     )
   }
 
-  private predicate rankedCmdSubstitutionReplacements(int i, string old, string new) {
-    old = rank[i](string old2 | this.cmdSubstitutionReplacement(old2, _, _) | old2) and
-    this.cmdSubstitutionReplacement(old, new, _)
+  /**
+   * Holds if `command` is a command in backticks `` `...` `` in
+   * the line at `lineIndex` in the shell script,
+   * and `id` is a unique identifier for this command.
+   */
+  private predicate commandInBackticks(int lineIndex, string command, string id) {
+    exists(int occurrenceIndex, int occurrenceOffset |
+      command =
+        this.lineProducer(lineIndex)
+            .regexpFind("\\`[^\\`]+\\`", occurrenceIndex, occurrenceOffset)
+            // trim leading backtick - TODO do this in first regex
+            .regexpReplaceAll("^\\`", "")
+            // trim trailing backtick - TODO do this in first regex
+            .regexpReplaceAll("\\`$", "") and
+      id = "cmd:" + lineIndex + ":" + occurrenceIndex + ":" + occurrenceOffset
+    )
+  }
+
+  private predicate rankedCmdSubstitutionReplacements(int i, string command, string commandId) {
+    // rank commands by their unique IDs
+    commandId = rank[i](string c, string id | this.cmdSubstitutionReplacement(c, id, _) | id) and
+    // since we cannot output (command, ID) tuples from the rank operation,
+    // we need to work out the specific command associated with the resulting ID
+    this.cmdSubstitutionReplacement(command, commandId, _)
   }
 
   private predicate doReplaceCmdSubstitutions(int line, int round, string old, string new) {
@@ -64,31 +93,56 @@ class BashShellScript extends ShellScript {
     this.cmdSubstitutionReplacement(result, _, i)
   }
 
+  /**
+   * Holds if `quotedStr` is a string in double quotes in
+   * the line at `lineIndex` in the shell script,
+   * and `id` is a unique identifier for this quoted string.
+   */
+  private predicate doubleQuotedString(int lineIndex, string quotedStr, string id) {
+    exists(int occurrenceIndex, int occurrenceOffset |
+      // double quoted string
+      quotedStr =
+        this.cmdSubstitutedLineProducer(lineIndex)
+            .regexpFind("\"((?:[^\"\\\\]|\\\\.)*)\"", occurrenceIndex, occurrenceOffset) and
+      id =
+        "qstr:" + lineIndex + ":" + occurrenceIndex + ":" + occurrenceOffset + ":" +
+          quotedStr.length() + ":" + quotedStr.regexpReplaceAll("[^a-zA-Z0-9]", "")
+    )
+  }
+
+  /**
+   * Holds if `quotedStr` is a string in single quotes in
+   * the line at `lineIndex` in the shell script,
+   * and `id` is a unique identifier for this quoted string.
+   */
+  private predicate singleQuotedString(int lineIndex, string quotedStr, string id) {
+    exists(int occurrenceIndex, int occurrenceOffset |
+      // single quoted string
+      quotedStr =
+        this.cmdSubstitutedLineProducer(lineIndex)
+            .regexpFind("'((?:\\\\.|[^'\\\\])*)'", occurrenceIndex, occurrenceOffset) and
+      id =
+        "qstr:" + lineIndex + ":" + occurrenceIndex + ":" + occurrenceOffset + ":" +
+          quotedStr.length() + ":" + quotedStr.regexpReplaceAll("[^a-zA-Z0-9]", "")
+    )
+  }
+
   private predicate quotedStringReplacement(string quotedStr, string id) {
-    exists(string line, int k | line = this.cmdSubstitutedLineProducer(k) |
-      exists(int i, int j |
-        // double quoted string
-        quotedStr = line.regexpFind("\"((?:[^\"\\\\]|\\\\.)*)\"", i, j) and
-        id =
-          "qstr:" + k + ":" + i + ":" + j + ":" + quotedStr.length() + ":" +
-            quotedStr.regexpReplaceAll("[^a-zA-Z0-9]", "")
-      )
+    exists(int lineIndex |
+      this.doubleQuotedString(lineIndex, quotedStr, id)
       or
-      exists(int i, int j |
-        // single quoted string
-        quotedStr = line.regexpFind("'((?:\\\\.|[^'\\\\])*)'", i, j) and
-        id =
-          "qstr:" + k + ":" + i + ":" + j + ":" + quotedStr.length() + ":" +
-            quotedStr.regexpReplaceAll("[^a-zA-Z0-9]", "")
-      )
+      this.singleQuotedString(lineIndex, quotedStr, id)
     ) and
     // Only do this for strings that might otherwise disrupt subsequent parsing
     quotedStr.regexpMatch("[\"'].*[$\n\r'\"" + Bash::separator() + "].*[\"']")
   }
 
-  private predicate rankedQuotedStringReplacements(int i, string old, string new) {
-    old = rank[i](string old2 | this.quotedStringReplacement(old2, _) | old2) and
-    this.quotedStringReplacement(old, new)
+  private predicate rankedQuotedStringReplacements(int i, string quotedString, string quotedStringId) {
+    // rank quoted strings by their nearly-unique IDs
+    quotedStringId = rank[i](string s, string id | this.quotedStringReplacement(s, id) | id) and
+    // since we cannot output (string, ID) tuples from the rank operation,
+    // we need to work out the specific string associated with the resulting ID
+    this.quotedStringReplacement(quotedString, quotedStringId)
   }
 
   private predicate doReplaceQuotedStrings(int line, int round, string old, string new) {

actions/ql/lib/codeql/actions/security/OutputClobberingQuery.qll

@@ -214,6 +214,8 @@ private module OutputClobberingConfig implements DataFlow::ConfigSig {
       )
     )
   }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Tracks flow of unsafe user input that is used to construct and evaluate an environment variable. */

actions/ql/lib/codeql/actions/security/RequestForgeryQuery.qll

@@ -16,6 +16,8 @@ private module RequestForgeryConfig implements DataFlow::ConfigSig {
   predicate isSource(DataFlow::Node source) { source instanceof RemoteFlowSource }
 
   predicate isSink(DataFlow::Node sink) { sink instanceof RequestForgerySink }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Tracks flow of unsafe user input that is used to construct and evaluate a system command. */

actions/ql/lib/codeql/actions/security/SecretExfiltrationQuery.qll

@@ -15,6 +15,8 @@ private module SecretExfiltrationConfig implements DataFlow::ConfigSig {
   predicate isSource(DataFlow::Node source) { source instanceof RemoteFlowSource }
 
   predicate isSink(DataFlow::Node sink) { sink instanceof SecretExfiltrationSink }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
 }
 
 /** Tracks flow of unsafe user input that is used in a context where it may lead to a secret exfiltration. */

actions/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/actions-all
-version: 0.4.10-dev
+version: 0.4.12-dev
 library: true
 warnOnImplicitThis: true
 dependencies:

actions/ql/src/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.6.3
+
+No user-facing changes.
+
+## 0.6.2
+
+### Minor Analysis Improvements
+
+* The query `actions/missing-workflow-permissions` is now aware of the minimal permissions needed for the actions `deploy-pages`, `delete-package-versions`, `ai-inference`. This should lead to better alert messages and better fix suggestions.
+
 ## 0.6.1
 
 No user-facing changes.