Benchmark and speed up to first stage of binning.

.copier-answers.yml

@@ -1,5 +1,5 @@
 # Changes here will be overwritten by Copier
-_commit: v2.0.1
+_commit: v2.0.2
 _src_path: gh:lincc-frameworks/python-project-template
 author_email: [email protected]
 author_name: LINCC Frameworks
@@ -11,7 +11,7 @@ enforce_style:
 - isort
 failure_notification:
 - slack
-include_benchmarks: false
+include_benchmarks: true
 include_docs: true
 include_notebooks: true
 mypy_type_checking: basic

.github/ISSUE_TEMPLATE/1-bug_report.md

@@ -13,5 +13,5 @@ assignees: ''
 Please check the following:
 
 - [ ] I have described the situation in which the bug arose, including what code was executed, information about my environment, and any applicable data others will need to reproduce the problem.
-- [ ] I have included available evidence of the unexpected behavior (including error messages, screenshots, and/or plots) as well as a descriprion of what I expected instead.
-- [ ] If I have a solution in mind, I have provided an explanation and/or pseudocode and/or task list.
+- [ ] I have included available evidence of the unexpected behavior (including error messages, screenshots, and/or plots) as well as a description of what I expected instead.
+- [ ] If I have a solution in mind, I have provided an explanation and/or pseudocode and/or task list.

.github/workflows/asv-main.yml

@@ -0,0 +1,101 @@
+# This workflow will run benchmarks with airspeed velocity (asv), 
+# store the new results in the "benchmarks" branch and publish them
+# to a dashboard on GH Pages.
+
+name: Run ASV benchmarks for main
+
+on:
+  push:
+    branches: [ main ]
+
+env:
+  PYTHON_VERSION: "3.10"
+  WORKING_DIR: ${{ github.workspace }}/benchmarks
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+
+  setup-python:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Cache Python ${{ env.PYTHON_VERSION }}
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: python-${{ env.PYTHON_VERSION }}
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+  asv-main:
+    runs-on: ubuntu-latest
+    needs: setup-python
+
+    permissions:
+      contents: write
+
+    defaults:
+      run:
+        working-directory: ${{ env.WORKING_DIR }}
+
+    steps:
+      - name: Checkout main branch of the repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Cache Python ${{ env.PYTHON_VERSION }}
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: python-${{ env.PYTHON_VERSION }}
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          python -m pip install --upgrade pip
+          pip install asv==0.6.1 virtualenv tabulate
+
+      - name: Configure git
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+      - name: Create ASV machine config file
+        run: asv machine --machine gh-runner --yes
+
+      - name: Fetch previous results from the "benchmarks" branch
+        run: |
+          if git ls-remote --exit-code origin benchmarks > /dev/null 2>&1; then
+            git merge origin/benchmarks \
+              --allow-unrelated-histories \
+              --no-commit
+            mv ../_results .
+          fi
+
+      - name: Run ASV for the main branch
+        run: asv run ALL --skip-existing --verbose || true
+
+      - name: Submit new results to the "benchmarks" branch
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          branch: benchmarks
+          folder: ${{ env.WORKING_DIR }}/_results
+          target-folder: _results
+
+      - name: Generate dashboard HTML
+        run: |
+          asv show
+          asv publish
+
+      - name: Deploy to Github pages
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          branch: gh-pages
+          folder: ${{ env.WORKING_DIR }}/_html

.github/workflows/asv-nightly.yml

@@ -0,0 +1,93 @@
+# This workflow will run daily at 06:45.
+# It will run benchmarks with airspeed velocity (asv)
+# and compare performance with the previous nightly build.
+
+name: Run benchmarks nightly job
+
+on:
+  schedule:
+    - cron: 45 6 * * *
+  workflow_dispatch:
+
+env:
+  PYTHON_VERSION: "3.10"
+  WORKING_DIR: ${{ github.workspace }}/benchmarks
+  NIGHTLY_HASH_FILE: nightly-hash
+
+jobs:
+
+  asv-nightly:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: ${{ env.WORKING_DIR }}
+
+    steps:
+      - name: Checkout main branch of the repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Cache Python ${{ env.PYTHON_VERSION }}
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: python-${{ env.PYTHON_VERSION }}
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          python -m pip install --upgrade pip
+          pip install asv==0.6.1 virtualenv
+
+      - name: Configure git
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+      - name: Create ASV machine config file
+        run: asv machine --machine gh-runner --yes
+
+      - name: Fetch previous results from the "benchmarks" branch
+        run: |
+          if git ls-remote --exit-code origin benchmarks > /dev/null 2>&1; then
+            git merge origin/benchmarks \
+              --allow-unrelated-histories \
+              --no-commit
+            mv ../_results .
+          fi
+
+      - name: Get nightly dates under comparison
+        id: nightly-dates
+        run: |
+          echo "yesterday=$(date -d yesterday +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+          echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+
+      - name: Use last nightly commit hash from cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.WORKING_DIR }}
+          key: nightly-results-${{ steps.nightly-dates.outputs.yesterday }}
+
+      - name: Run comparison of main against last nightly build
+        run: |
+          HASH_FILE=${{ env.NIGHTLY_HASH_FILE }}
+          CURRENT_HASH=${{ github.sha }}
+          if [ -f $HASH_FILE ]; then          
+            PREV_HASH=$(cat $HASH_FILE)
+            asv continuous $PREV_HASH $CURRENT_HASH --verbose || true
+            asv compare $PREV_HASH $CURRENT_HASH --sort ratio --verbose
+          fi
+          echo $CURRENT_HASH > $HASH_FILE
+
+      - name: Update last nightly hash in cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.WORKING_DIR }}
+          key: nightly-results-${{ steps.nightly-dates.outputs.today }}

.github/workflows/asv-pr.yml

@@ -0,0 +1,86 @@
+# This workflow will run benchmarks with airspeed velocity (asv) for pull requests.
+# It will compare the performance of the main branch with the performance of the merge
+# with the new changes. It then publishes a comment with this assessment by triggering
+# the publish-benchmarks-pr workflow.
+# Based on https://securitylab.github.com/research/github-actions-preventing-pwn-requests/.
+name: Run benchmarks for PR
+
+on:
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  PYTHON_VERSION: "3.10"
+  WORKING_DIR: ${{ github.workspace }}/benchmarks
+  ARTIFACTS_DIR: ${{ github.workspace }}/artifacts
+
+jobs:
+  setup-python:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Cache Python ${{ env.PYTHON_VERSION }}
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: python-${{ env.PYTHON_VERSION }}
+    - name: Set up Python ${{ env.PYTHON_VERSION }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+  asv-pr:
+    runs-on: ubuntu-latest
+    needs: setup-python
+    defaults:
+      run:
+        working-directory: ${{ env.WORKING_DIR }}
+    steps:
+    - name: Checkout PR branch of the repository
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Display Workflow Run Information
+      run: |
+        echo "Workflow Run ID: ${{ github.run_id }}"
+    - name: Cache Python ${{ env.PYTHON_VERSION }}
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: python-${{ env.PYTHON_VERSION }}
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        python -m pip install --upgrade pip
+        pip install asv==0.6.1 virtualenv tabulate lf-asv-formatter
+    - name: Make artifacts directory
+      run: mkdir -p ${{ env.ARTIFACTS_DIR }}
+    - name: Save pull request number
+      run: echo ${{ github.event.pull_request.number }} > ${{ env.ARTIFACTS_DIR }}/pr
+    - name: Get current job logs URL
+      uses: Tiryoh/gha-jobid-action@v1
+      id: jobs
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        job_name: ${{ github.job }}
+    - name: Create ASV machine config file
+      run: asv machine --machine gh-runner --yes
+    - name: Save comparison of PR against main branch
+      run: |
+        git remote add upstream https://github.com/${{ github.repository }}.git
+        git fetch upstream
+        asv continuous upstream/main HEAD --verbose || true
+        asv compare upstream/main HEAD --sort ratio --verbose | tee output
+        python -m lf_asv_formatter --asv_version "$(echo asv --version)"
+        printf "\n\nClick [here]($STEP_URL) to view all benchmarks." >> output
+        mv output ${{ env.ARTIFACTS_DIR }}
+      env:
+        STEP_URL: "${{ steps.jobs.outputs.html_url }}#step:11:1"
+    - name: Upload artifacts (PR number and benchmarks output)
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmark-artifacts
+        path: ${{ env.ARTIFACTS_DIR }}

.github/workflows/publish-benchmarks-pr.yml

@@ -0,0 +1,53 @@
+# This workflow publishes a benchmarks comment on a pull request. It is triggered after the
+# benchmarks are computed in the asv-pr workflow. This separation of concerns allows us limit
+# access to the target repository private tokens and secrets, increasing the level of security.
+# Based on https://securitylab.github.com/research/github-actions-preventing-pwn-requests/.
+name: Publish benchmarks comment to PR
+
+on:
+  workflow_run:
+    workflows: ["Run benchmarks for PR"]
+    types: [completed]
+
+jobs:
+  upload-pr-comment:
+    runs-on: ubuntu-latest
+    if: >
+      github.event.workflow_run.event == 'pull_request' &&
+      github.event.workflow_run.conclusion == 'success'
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+    - name: Display Workflow Run Information
+      run: |
+        echo "Workflow Run ID: ${{ github.event.workflow_run.id }}"
+        echo "Head SHA: ${{ github.event.workflow_run.head_sha }}"
+        echo "Head Branch: ${{ github.event.workflow_run.head_branch }}"
+        echo "Conclusion: ${{ github.event.workflow_run.conclusion }}"
+        echo "Event: ${{ github.event.workflow_run.event }}"
+    - name: Download artifact
+      uses: dawidd6/action-download-artifact@v3
+      with:
+        name: benchmark-artifacts
+        run_id: ${{ github.event.workflow_run.id }}
+    - name: Extract artifacts information
+      id: pr-info
+      run: |
+        printf "PR number: $(cat pr)\n"
+        printf "Output:\n$(cat output)"
+        printf "pr=$(cat pr)" >> $GITHUB_OUTPUT
+    - name: Find benchmarks comment
+      uses: peter-evans/find-comment@v3
+      id: find-comment
+      with:
+        issue-number: ${{ steps.pr-info.outputs.pr }}
+        comment-author: 'github-actions[bot]'
+        body-includes: view all benchmarks
+    - name: Create or update benchmarks comment
+      uses: peter-evans/create-or-update-comment@v4
+      with:
+        comment-id: ${{ steps.find-comment.outputs.comment-id }}
+        issue-number: ${{ steps.pr-info.outputs.pr }}
+        body-path: output
+        edit-mode: replace

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
     # This hook should always pass. It will print a message if the local version 
     # is out of date.
   - repo: https://github.com/lincc-frameworks/pre-commit-hooks
-    rev: v0.1.1
+    rev: v0.1.2
     hooks:
       - id: check-lincc-frameworks-template-version
         name: Check template version