Enhance CI workflows (#914)

1) Cache the Python pip dependencies
2) Run on both x64 and arm64 architectures
3) Allow manual invocation of workflow, for testing
4) Make it cancel previously-started runs when a new run is started
5) Normalize quotes to using double quotes
6) Add some debugging output

.github/workflows/ci.yml

@@ -1,4 +1,14 @@
-name: Continuous Integration
+# Summary: OpenFermion continuous integration status checks.
+#
+# This workflow runs various tests to verify that changes to the OpenFermion
+# codebase pass validation and conform to project format and style standards.
+# It triggers on certain events such as pull requests and merge-queue merges,
+# and can also be invoked manually via the "Run workflow" button at
+# https://github.com/quantumlib/OpenFermion/actions/workflows/ci.yaml
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+name: Continuous integration checks
+run-name: Continuous integration checks triggered by ${{github.event_name}}
 
 on:
   pull_request:
@@ -14,68 +24,146 @@ on:
     branches:
       - master
 
+  # Allow manual invocation.
+  workflow_dispatch:
+    inputs:
+      python_ver:
+        description: Version of Python to use
+        type: string
+        default: "3.10.11"
+
+concurrency:
+  # Cancel any previously-started but still active runs on the same branch.
+  cancel-in-progress: true
+  group: ${{github.workflow}}-${{github.event.pull_request.number||github.ref}}
+
+env:
+  # Default Python version to use.
+  python_ver: "3.10.11"
+
+  # Files listing dependencies we install using pip in the various jobs below.
+  # This is used by setup-python to check whether its cache needs updating.
+  python_dep_files: >-
+    dev_tools/requirements/envs/format.env.txt
+    dev_tools/requirements/envs/mypy.env.txt
+    dev_tools/requirements/envs/pylint.env.txt
+    dev_tools/requirements/envs/pytest-extra.env.txt
+    dev_tools/requirements/envs/pytest.env.txt
+    dev_tools/requirements/max_compat/pytest-max-compat.env.txt
+
 jobs:
+  Setup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out a copy of the OpenFermion git repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python with caching of pip dependencies
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{inputs.python_ver || env.python_ver}}
+          architecture: "x64"
+          cache: pip
+          cache-dependency-path: ${{env.python_dep_files}}
+
+      - name: Install Python requirements
+        run: |
+          set -x
+          for file in ${{env.python_dep_files}}; do
+            pip install -r $file
+          done
+          set +x
+          echo "::group::List of installed pip packages and their versions"
+          pip list
+          echo "::endgroup::"
+
   format:
     name: Format check
+    needs: Setup
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
-          architecture: 'x64'
+          python-version: ${{inputs.python_ver || env.python_ver}}
+          architecture: "x64"
           cache: pip
+          cache-dependency-path: ${{env.python_dep_files}}
+
       - name: Install requirements
         run: pip install -r dev_tools/requirements/envs/format.env.txt
+
       - name: Format
         run: check/format-incremental
+
   mypy:
     name: Type check
+    needs: Setup
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
-          architecture: 'x64'
+          python-version: ${{inputs.python_ver || env.python_ver}}
+          architecture: "x64"
           cache: pip
+          cache-dependency-path: ${{env.python_dep_files}}
+
       - name: Install requirements
         run: pip install -r dev_tools/requirements/envs/mypy.env.txt
+
       - name: Type check
         run: check/mypy
+
   pylint:
     name: Lint check
+    needs: Setup
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
-          architecture: 'x64'
+          python-version: ${{inputs.python_ver || env.python_ver}}
+          architecture: "x64"
           cache: pip
+          cache-dependency-path: ${{env.python_dep_files}}
+
       - name: Install requirements
         run: pip install -r dev_tools/requirements/envs/pylint.env.txt
+
       - name: Lint
         run: check/pylint
+
   pytest-max-compat:
     name: Pytest max compat
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v4
+
+        # Note: deliberately not using our Python cache here b/c this runs
+        # a different version of Python.
       - uses: actions/setup-python@v5
         with:
           python-version: 3.8
-          cache: pip
+
       - name: Install requirements
         run: |
           pip install -r dev_tools/requirements/max_compat/pytest-max-compat.env.txt
+
       - name: Pytest check
         run: check/pytest
         shell: bash
+
   pytest:
     name: Pytest
+    needs: Setup
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -84,19 +172,25 @@ jobs:
       fail-fast: false
     steps:
       - uses: actions/checkout@v4
+
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: ${{inputs.python_ver || env.python_ver}}
           cache: pip
+          cache-dependency-path: ${{env.python_dep_files}}
+
       - name: Install requirements
         run: |
           pip install -r dev_tools/requirements/envs/pytest.env.txt
           pip install cirq-core==${{matrix.cirq-version}}
+
       - name: Pytest check
         run: check/pytest
         shell: bash
+
   pytest-extra:
     name: Pytest extra
+    needs: Setup
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -105,29 +199,39 @@ jobs:
       fail-fast: false
     steps:
       - uses: actions/checkout@v4
+
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: ${{inputs.python_ver || env.python_ver}}
           cache: pip
+          cache-dependency-path: ${{env.python_dep_files}}
+
       - name: Install requirements
         run: |
           pip install -r dev_tools/requirements/envs/pytest-extra.env.txt
           pip install cirq-core==${{matrix.cirq-version}}
+
       - name: Pytest check resources
-        run: check/pytest -m 'not slow' src/openfermion/resource_estimates
+        run: check/pytest -m "not slow" src/openfermion/resource_estimates
         shell: bash
+
   coverage:
     name: Coverage check
+    needs: Setup
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: ${{inputs.python_ver || env.python_ver}}
           cache: pip
+          cache-dependency-path: ${{env.python_dep_files}}
+
       - name: Install requirements
         run: pip install -r dev_tools/requirements/envs/pytest.env.txt
+
       - name: Coverage check
         run: check/pytest-and-incremental-coverage

.github/workflows/nightly.yml

@@ -1,28 +1,106 @@
-name: Nightly
+# Summary: OpenFermion nightly tests.
+#
+# This workflow runs nightly to run tests on the OpenFermion codebase.
+# It can also be invoked manually via the "Run workflow" button at
+# https://github.com/quantumlib/OpenFermion/actions/workflows/nightly.yaml
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+name: Nightly full tests
 
 on:
   schedule:
-    - cron: "0 0 * * *"
+    - cron: "15 4 * * *"
+
+  # Allow manual invocation.
+  workflow_dispatch:
+
+concurrency:
+  # Cancel any previously-started but still active runs on the same branch.
+  cancel-in-progress: true
+  group: ${{github.workflow}}-${{github.event.pull_request.number||github.ref}}
 
 jobs:
-  cirq-next:
-    name: Cirq Pre-release
-    runs-on: ubuntu-latest
+  cirq-stable:
+    name: Test current stable Cirq release
+    runs-on: ${{matrix.os}}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.10.12", "3.11.9", "3.12.8"]
+        arch: [x64, arm64]
+        exclude:
+          - os: windows-latest
+            arch: arm64
+      fail-fast: false
+    steps:
+      - name: Check out a copy of the OpenFermion git repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{matrix.python-version}}
+        uses: actions/setup-python@v5
+        id: cache
+        with:
+          python-version: ${{matrix.python-version}}
+          architecture: ${{matrix.arch}}
+          cache: pip
+          cache-dependency-path: dev_tools/requirements/envs/pytest.env.txt
+
+      - if: steps.cache.outputs.cache-hit != 'true'
+        name: Install OpenFermion Python requirements
+        run: |
+          pip install -r dev_tools/requirements/envs/pytest.env.txt
+          echo "::group::List of installed pip packages and their versions"
+          pip list
+          echo "::endgroup::"
+
+        # The pytest.env.txt file includes cirq-core, but it's a pinned
+        # version. Here we install the latest released stable version, which
+        # may or may not be the same.
+      - name: Install cirq-core (current stable version)
+        run: |
+          pip install -U cirq-core
+
+      - name: Run Pytest
+        run: |
+          check/pytest
+
+  cirq-dev:
+    name: Test latest Cirq pre-release
+    runs-on: ${{matrix.os}}
     strategy:
       matrix:
-        os: [ ubuntu-latest, macos-latest, windows-latest ]
-        python-version: [ '3.10' ]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.10.12", "3.11.9", "3.12.8"]
+        arch: ["x64", "arm64"]
+        exclude:
+          - os: windows-latest
+            arch: arm64
       fail-fast: false
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - name: Check out a copy of the OpenFermion git repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{matrix.python-version}}
+        id: cache
+        uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{matrix.python-version}}
+          architecture: ${{matrix.arch}}
           cache: pip
-      - name: Install requirements
+          cache-dependency-path: dev_tools/requirements/envs/pytest.env.txt
+
+      - if: steps.cache.outputs.cache-hit != 'true'
+        name: Install OpenFermion Python requirements
         run: |
           pip install -r dev_tools/requirements/envs/pytest.env.txt
+          echo "::group::List of installed pip packages and their versions"
+          pip list
+          echo "::endgroup::"
+
+      - name: Install cirq-core (pre-release version)
+        run: |
           pip install -U cirq-core --pre
-      - name: Pytest check
-        run: check/pytest
-        shell: bash
+
+      - name: Run Pytest
+        run: |
+          check/pytest