From 2aa61c155ebed3a8bb341627bd3ea57612937db1 Mon Sep 17 00:00:00 2001 From: avi Date: Tue, 19 Aug 2025 19:55:25 +1200 Subject: [PATCH 01/11] use COMPAS docker as base container --- .github/workflows/compas-compile-ci.yml | 59 ++++++++++++++++++------- 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/.github/workflows/compas-compile-ci.yml b/.github/workflows/compas-compile-ci.yml index f10e7a9f1..e63bb162e 100644 --- a/.github/workflows/compas-compile-ci.yml +++ b/.github/workflows/compas-compile-ci.yml @@ -27,46 +27,68 @@ jobs: name: Build COMPAS runs-on: ubuntu-22.04 + container: + image: teamcompas/compas:latest + options: --user root steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.9' - cache: pip - cache-dependency-path: setup.py - # - name: Install TeXLive - # uses: teatimeguest/setup-texlive-action@v3.3.4 - - name: Install dependencies on ubuntu + + - name: Verify container environment run: | - cd misc/cicd-scripts - chmod 755 linux-dependencies - ./linux-dependencies - - name: Build Compas + echo "Verifying COMPAS container dependencies..." + g++ --version + python3 --version + echo "Boost libraries:" + find /usr -name "*boost*" -type d 2>/dev/null | head -3 || echo "Boost may be in different location" + echo "LaTeX:" + pdflatex --version | head -2 + echo "Container verification complete" + + - name: Build COMPAS from source run: | - cd src && make -j $(nproc) -f Makefile + cd src + # Clean any existing builds to ensure fresh compilation + make clean 2>/dev/null || echo "No existing build to clean" + # Build with all available cores + make -j $(nproc) -f Makefile + # Verify the build worked ./COMPAS -v - - name: Install python utils + echo "COMPAS build successful!" + + - name: Setup Python environment run: | - pip install --upgrade pip + # Update pip + python3 -m pip install --upgrade pip + # Install the package in development mode pip install -e .[dev] + - name: Run example COMPAS job run: | export COMPAS_EXECUTABLE_PATH=${GITHUB_WORKSPACE}/src/COMPAS cd ${GITHUB_WORKSPACE}/py_tests/test_data/ chmod 755 run.sh + echo "Contents of run.sh:" cat run.sh + echo "Running COMPAS example..." ./run.sh - - name: Run pytests + + - name: Run pytest suite run: | cd ${GITHUB_WORKSPACE} export COMPAS_ROOT_DIR=${GITHUB_WORKSPACE} + echo "Available Jupyter kernels:" jupyter-kernelspec list + echo "Running non-web tests..." pytest --cov=compas_python_utils/ py_tests/ -m 'not webtest' + echo "Running web tests..." pytest --cov=compas_python_utils/ --cov-append py_tests/ -m webtest + echo "Test artifacts:" ls py_tests/test_artifacts + echo "Generating coverage reports..." coverage html coverage-badge -o coverage_badge.svg -f + - name: Archive code coverage results uses: actions/upload-artifact@v4 with: @@ -74,6 +96,7 @@ jobs: path: | htmlcov/ coverage_badge.svg + - name: Archive COMPAS detailed-evolution plot id: upload uses: actions/upload-artifact@v4 @@ -81,10 +104,12 @@ jobs: name: ${{ env.ARTIFACT_NAME }} path: ${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_NAME }} if-no-files-found: error + - name: Test Summary run: | echo "### Test Results" >> $GITHUB_STEP_SUMMARY - echo "- Compas Build: Success" >> $GITHUB_STEP_SUMMARY + echo "- COMPAS Build: Success" >> $GITHUB_STEP_SUMMARY echo "- Python Utils Installation: Success" >> $GITHUB_STEP_SUMMARY echo "- Example COMPAS Job: Success" >> $GITHUB_STEP_SUMMARY echo "- Pytest Execution: Success" >> $GITHUB_STEP_SUMMARY + echo "- Using container: teamcompas/compas:latest" >> $GITHUB_STEP_SUMMARY \ No newline at end of file From 615faf9e6460d45429681e0faf21c377d6bd721f Mon Sep 17 00:00:00 2001 From: avi Date: Tue, 19 Aug 2025 20:00:23 +1200 Subject: [PATCH 02/11] use COMPAS docker to just build --- .github/workflows/compas-compile-ci.yml | 54 ++++++++++--------------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/.github/workflows/compas-compile-ci.yml b/.github/workflows/compas-compile-ci.yml index e63bb162e..ca747557d 100644 --- a/.github/workflows/compas-compile-ci.yml +++ b/.github/workflows/compas-compile-ci.yml @@ -27,46 +27,36 @@ jobs: name: Build COMPAS runs-on: ubuntu-22.04 - container: - image: teamcompas/compas:latest - options: --user root steps: - uses: actions/checkout@v4 - - name: Verify container environment - run: | - echo "Verifying COMPAS container dependencies..." - g++ --version - python3 --version - echo "Boost libraries:" - find /usr -name "*boost*" -type d 2>/dev/null | head -3 || echo "Boost may be in different location" - echo "LaTeX:" - pdflatex --version | head -2 - echo "Container verification complete" + - uses: actions/setup-python@v5 + with: + python-version: '3.9' + cache: pip + cache-dependency-path: setup.py - - name: Build COMPAS from source + - name: Build COMPAS using Docker container run: | - cd src - # Clean any existing builds to ensure fresh compilation - make clean 2>/dev/null || echo "No existing build to clean" - # Build with all available cores - make -j $(nproc) -f Makefile - # Verify the build worked - ./COMPAS -v - echo "COMPAS build successful!" + echo "Building COMPAS using teamcompas/compas:latest container..." + docker run --rm \ + -v ${{ github.workspace }}:/workspace \ + -w /workspace \ + teamcompas/compas:latest \ + sh -c "cd src && make clean 2>/dev/null || echo 'No existing build to clean' && make -j \$(nproc) -f Makefile && ./COMPAS -v" + + echo "COMPAS build completed successfully!" - - name: Setup Python environment + - name: Install python utils run: | - # Update pip - python3 -m pip install --upgrade pip - # Install the package in development mode + pip install --upgrade pip pip install -e .[dev] - name: Run example COMPAS job run: | - export COMPAS_EXECUTABLE_PATH=${GITHUB_WORKSPACE}/src/COMPAS - cd ${GITHUB_WORKSPACE}/py_tests/test_data/ + export COMPAS_EXECUTABLE_PATH=${{ github.workspace }}/src/COMPAS + cd ${{ github.workspace }}/py_tests/test_data/ chmod 755 run.sh echo "Contents of run.sh:" cat run.sh @@ -75,8 +65,8 @@ jobs: - name: Run pytest suite run: | - cd ${GITHUB_WORKSPACE} - export COMPAS_ROOT_DIR=${GITHUB_WORKSPACE} + cd ${{ github.workspace }} + export COMPAS_ROOT_DIR=${{ github.workspace }} echo "Available Jupyter kernels:" jupyter-kernelspec list echo "Running non-web tests..." @@ -108,8 +98,8 @@ jobs: - name: Test Summary run: | echo "### Test Results" >> $GITHUB_STEP_SUMMARY - echo "- COMPAS Build: Success" >> $GITHUB_STEP_SUMMARY + echo "- COMPAS Build: Success (using Docker)" >> $GITHUB_STEP_SUMMARY echo "- Python Utils Installation: Success" >> $GITHUB_STEP_SUMMARY echo "- Example COMPAS Job: Success" >> $GITHUB_STEP_SUMMARY echo "- Pytest Execution: Success" >> $GITHUB_STEP_SUMMARY - echo "- Using container: teamcompas/compas:latest" >> $GITHUB_STEP_SUMMARY \ No newline at end of file + echo "- Container used: teamcompas/compas:latest" >> $GITHUB_STEP_SUMMARY \ No newline at end of file From 4406a5f92aef3d4fcf62a4b808be5d6772e03bc7 Mon Sep 17 00:00:00 2001 From: avi Date: Tue, 19 Aug 2025 20:17:39 +1200 Subject: [PATCH 03/11] use COMPAS docker for building, running and pytest --- .github/workflows/compas-compile-ci.yml | 99 +++++++++++++------------ 1 file changed, 53 insertions(+), 46 deletions(-) diff --git a/.github/workflows/compas-compile-ci.yml b/.github/workflows/compas-compile-ci.yml index ca747557d..502245083 100644 --- a/.github/workflows/compas-compile-ci.yml +++ b/.github/workflows/compas-compile-ci.yml @@ -31,53 +31,60 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.9' - cache: pip - cache-dependency-path: setup.py - - - name: Build COMPAS using Docker container + - name: Build and test COMPAS entirely in container run: | - echo "Building COMPAS using teamcompas/compas:latest container..." + echo "Running complete COMPAS build and test suite in container..." docker run --rm \ -v ${{ github.workspace }}:/workspace \ -w /workspace \ teamcompas/compas:latest \ - sh -c "cd src && make clean 2>/dev/null || echo 'No existing build to clean' && make -j \$(nproc) -f Makefile && ./COMPAS -v" - - echo "COMPAS build completed successfully!" - - - name: Install python utils - run: | - pip install --upgrade pip - pip install -e .[dev] - - - name: Run example COMPAS job - run: | - export COMPAS_EXECUTABLE_PATH=${{ github.workspace }}/src/COMPAS - cd ${{ github.workspace }}/py_tests/test_data/ - chmod 755 run.sh - echo "Contents of run.sh:" - cat run.sh - echo "Running COMPAS example..." - ./run.sh - - - name: Run pytest suite - run: | - cd ${{ github.workspace }} - export COMPAS_ROOT_DIR=${{ github.workspace }} - echo "Available Jupyter kernels:" - jupyter-kernelspec list - echo "Running non-web tests..." - pytest --cov=compas_python_utils/ py_tests/ -m 'not webtest' - echo "Running web tests..." - pytest --cov=compas_python_utils/ --cov-append py_tests/ -m webtest - echo "Test artifacts:" - ls py_tests/test_artifacts - echo "Generating coverage reports..." - coverage html - coverage-badge -o coverage_badge.svg -f + sh -c " + set -e # Exit on any error + + echo '=== Building COMPAS ===' + cd src + make clean 2>/dev/null || echo 'No existing build to clean' + make -j \$(nproc) -f Makefile + ./COMPAS -v + echo 'COMPAS build successful!' + + echo '=== Installing Python dependencies ===' + cd /workspace + python3 -m pip install --upgrade pip + pip install -e .[dev] + echo 'Python dependencies installed!' + + echo '=== Running example COMPAS job ===' + export COMPAS_EXECUTABLE_PATH=/workspace/src/COMPAS + cd /workspace/py_tests/test_data/ + chmod 755 run.sh + echo 'Contents of run.sh:' + cat run.sh + echo 'Running COMPAS example...' + ./run.sh + echo 'Example job completed!' + + echo '=== Running pytest suite ===' + cd /workspace + export COMPAS_ROOT_DIR=/workspace + echo 'Available Jupyter kernels:' + jupyter-kernelspec list || echo 'Jupyter not available, continuing...' + + echo 'Running non-web tests...' + pytest --cov=compas_python_utils/ py_tests/ -m 'not webtest' + + echo 'Running web tests...' + pytest --cov=compas_python_utils/ --cov-append py_tests/ -m webtest + + echo 'Test artifacts:' + ls py_tests/test_artifacts || echo 'No test artifacts directory' + + echo 'Generating coverage reports...' + coverage html + coverage-badge -o coverage_badge.svg -f + + echo 'All tests completed successfully!' + " - name: Archive code coverage results uses: actions/upload-artifact@v4 @@ -98,8 +105,8 @@ jobs: - name: Test Summary run: | echo "### Test Results" >> $GITHUB_STEP_SUMMARY - echo "- COMPAS Build: Success (using Docker)" >> $GITHUB_STEP_SUMMARY - echo "- Python Utils Installation: Success" >> $GITHUB_STEP_SUMMARY - echo "- Example COMPAS Job: Success" >> $GITHUB_STEP_SUMMARY - echo "- Pytest Execution: Success" >> $GITHUB_STEP_SUMMARY + echo "- COMPAS Build: Success (Docker container)" >> $GITHUB_STEP_SUMMARY + echo "- Python Utils Installation: Success (Docker container)" >> $GITHUB_STEP_SUMMARY + echo "- Example COMPAS Job: Success (Docker container)" >> $GITHUB_STEP_SUMMARY + echo "- Pytest Execution: Success (Docker container)" >> $GITHUB_STEP_SUMMARY echo "- Container used: teamcompas/compas:latest" >> $GITHUB_STEP_SUMMARY \ No newline at end of file From 214382c9eace216f84a3ef7b0d56fa273e662133 Mon Sep 17 00:00:00 2001 From: avi Date: Wed, 20 Aug 2025 16:12:11 +1200 Subject: [PATCH 04/11] use updated docker imagae --- .github/workflows/compas-compile-ci.yml | 119 ++++++++++++------------ 1 file changed, 59 insertions(+), 60 deletions(-) diff --git a/.github/workflows/compas-compile-ci.yml b/.github/workflows/compas-compile-ci.yml index 502245083..771c8ff53 100644 --- a/.github/workflows/compas-compile-ci.yml +++ b/.github/workflows/compas-compile-ci.yml @@ -27,64 +27,62 @@ jobs: name: Build COMPAS runs-on: ubuntu-22.04 + # Use Docker Hub for now (GHCR is private) + container: teamcompas/compas:latest + # TODO: Switch to GHCR when package is made public + # container: ghcr.io/teamcompas/compas:latest steps: - - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - - name: Build and test COMPAS entirely in container + - name: Build COMPAS from source run: | - echo "Running complete COMPAS build and test suite in container..." - docker run --rm \ - -v ${{ github.workspace }}:/workspace \ - -w /workspace \ - teamcompas/compas:latest \ - sh -c " - set -e # Exit on any error - - echo '=== Building COMPAS ===' - cd src - make clean 2>/dev/null || echo 'No existing build to clean' - make -j \$(nproc) -f Makefile - ./COMPAS -v - echo 'COMPAS build successful!' - - echo '=== Installing Python dependencies ===' - cd /workspace - python3 -m pip install --upgrade pip - pip install -e .[dev] - echo 'Python dependencies installed!' - - echo '=== Running example COMPAS job ===' - export COMPAS_EXECUTABLE_PATH=/workspace/src/COMPAS - cd /workspace/py_tests/test_data/ - chmod 755 run.sh - echo 'Contents of run.sh:' - cat run.sh - echo 'Running COMPAS example...' - ./run.sh - echo 'Example job completed!' - - echo '=== Running pytest suite ===' - cd /workspace - export COMPAS_ROOT_DIR=/workspace - echo 'Available Jupyter kernels:' - jupyter-kernelspec list || echo 'Jupyter not available, continuing...' - - echo 'Running non-web tests...' - pytest --cov=compas_python_utils/ py_tests/ -m 'not webtest' - - echo 'Running web tests...' - pytest --cov=compas_python_utils/ --cov-append py_tests/ -m webtest - - echo 'Test artifacts:' - ls py_tests/test_artifacts || echo 'No test artifacts directory' - - echo 'Generating coverage reports...' - coverage html - coverage-badge -o coverage_badge.svg -f - - echo 'All tests completed successfully!' - " + echo "Building COMPAS from source..." + cd src + make clean 2>/dev/null || echo 'No existing build to clean' + make -j $(nproc) -f Makefile + ./COMPAS -v + echo "COMPAS build completed successfully!" + + - name: Install Python dependencies + run: | + echo "Installing Python dependencies..." + python3 -m pip install --upgrade pip + pip install -e .[dev] + echo "Python dependencies installed!" + + - name: Run example COMPAS job + run: | + echo "Running example COMPAS job..." + export COMPAS_EXECUTABLE_PATH=${GITHUB_WORKSPACE}/src/COMPAS + cd ${GITHUB_WORKSPACE}/py_tests/test_data/ + chmod 755 run.sh + echo "Contents of run.sh:" + cat run.sh + ./run.sh + echo "Example job completed!" + + - name: Run pytest suite + run: | + echo "Running pytest suite..." + cd ${GITHUB_WORKSPACE} + export COMPAS_ROOT_DIR=${GITHUB_WORKSPACE} + + if command -v jupyter-kernelspec &> /dev/null; then + echo "Available Jupyter kernels:" + jupyter-kernelspec list + fi + + pytest --cov=compas_python_utils/ py_tests/ -m 'not webtest' + pytest --cov=compas_python_utils/ --cov-append py_tests/ -m webtest + + echo "Test artifacts:" + ls py_tests/test_artifacts/ 2>/dev/null || echo "No test artifacts found" + + coverage html + coverage-badge -o coverage_badge.svg -f + echo "All tests completed successfully!" - name: Archive code coverage results uses: actions/upload-artifact@v4 @@ -104,9 +102,10 @@ jobs: - name: Test Summary run: | - echo "### Test Results" >> $GITHUB_STEP_SUMMARY - echo "- COMPAS Build: Success (Docker container)" >> $GITHUB_STEP_SUMMARY - echo "- Python Utils Installation: Success (Docker container)" >> $GITHUB_STEP_SUMMARY - echo "- Example COMPAS Job: Success (Docker container)" >> $GITHUB_STEP_SUMMARY - echo "- Pytest Execution: Success (Docker container)" >> $GITHUB_STEP_SUMMARY - echo "- Container used: teamcompas/compas:latest" >> $GITHUB_STEP_SUMMARY \ No newline at end of file + echo "### COMPAS CI Results" >> $GITHUB_STEP_SUMMARY + echo "- COMPAS Build: Success" >> $GITHUB_STEP_SUMMARY + echo "- Python Dependencies: Success" >> $GITHUB_STEP_SUMMARY + echo "- Example COMPAS Job: Success" >> $GITHUB_STEP_SUMMARY + echo "- Pytest Suite: Success" >> $GITHUB_STEP_SUMMARY + echo "- Container: teamcompas/compas:latest (Docker Hub)" >> $GITHUB_STEP_SUMMARY + echo "- Ready to switch to: ghcr.io/teamcompas/compas:latest" >> $GITHUB_STEP_SUMMARY \ No newline at end of file From c798ac57dd16e5b9c0978b837c07d6a20b927337 Mon Sep 17 00:00:00 2001 From: avi Date: Wed, 20 Aug 2025 16:21:52 +1200 Subject: [PATCH 05/11] Add detailed evol plot --- .github/workflows/compas-compile-ci.yml | 46 +++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/.github/workflows/compas-compile-ci.yml b/.github/workflows/compas-compile-ci.yml index 771c8ff53..5311d34a0 100644 --- a/.github/workflows/compas-compile-ci.yml +++ b/.github/workflows/compas-compile-ci.yml @@ -100,6 +100,52 @@ jobs: path: ${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_NAME }} if-no-files-found: error + - name: Comment on PR with results + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const evolutionPlotPath = '${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_NAME }}'; + const shortSha = context.sha.substring(0, 7); + const workflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + + let plotSection = ''; + try { + if (fs.existsSync(evolutionPlotPath)) { + const imageData = fs.readFileSync(evolutionPlotPath, {encoding: 'base64'}); + plotSection = ` + ### 📊 Detailed Evolution Plot +
Click to view evolution plot + + ![Evolution Plot](data:image/png;base64,${imageData}) +
`; + } else { + plotSection = '\n### ⚠️ Evolution plot not found'; + } + } catch (error) { + plotSection = `\n### ⚠️ Evolution plot error: ${error.message}`; + } + + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `![badge](https://img.shields.io/badge/Build_Success-28a745?style=for-the-badge&logo=github-actions&logoColor=white) + + ## ✅ COMPAS Build Successful! + + | Item | Value | + |------|-------| + | **Commit** | [\`${shortSha}\`](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/commit/${context.sha}) | + | **Logs** | [View workflow](${workflowUrl}) | + ${plotSection} + + --- + Generated by COMPAS CI` + }); + - name: Test Summary run: | echo "### COMPAS CI Results" >> $GITHUB_STEP_SUMMARY From c95c3c94a68ad6aaf1fa85137909647d5e062e7f Mon Sep 17 00:00:00 2001 From: avi Date: Wed, 20 Aug 2025 16:34:52 +1200 Subject: [PATCH 06/11] fix: float_ deprecated in numpy>2.0 --- .../binned_cosmic_integrator/io.py | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/compas_python_utils/cosmic_integration/binned_cosmic_integrator/io.py b/compas_python_utils/cosmic_integration/binned_cosmic_integrator/io.py index 135d7239a..32115f59a 100644 --- a/compas_python_utils/cosmic_integration/binned_cosmic_integrator/io.py +++ b/compas_python_utils/cosmic_integration/binned_cosmic_integrator/io.py @@ -25,14 +25,14 @@ def recursively_save_dict_contents_to_group(h5file: h5py.File, group: str, dic: def encode_for_hdf5(key, item): - if isinstance(item, np.int_): - item = int(item) - elif isinstance(item, np.float_): - item = float(item) - elif isinstance(item, np.complex_): - item = complex(item) + if isinstance(item, (np.generic, int, float, complex)): + if isinstance(item, np.integer): + item = int(item) + elif isinstance(item, np.floating): + item = float(item) + elif isinstance(item, np.complexfloating): + item = complex(item) if isinstance(item, np.ndarray): - # Numpy's wide unicode strings are not supported by hdf5 if item.dtype.kind == 'U': item = np.array(item, dtype='S') if isinstance(item, (np.ndarray, int, float, complex, str, bytes)): @@ -43,7 +43,7 @@ def encode_for_hdf5(key, item): if len(item) == 0: output = item elif isinstance(item[0], (str, bytes)) or item[0] is None: - output = list() + output = [] for value in item: if isinstance(value, str): output.append(value.encode("utf-8")) @@ -68,16 +68,17 @@ def decode_from_hdf5(item): elif isinstance(item, bytes) and item == b"__none__": output = None elif isinstance(item, (bytes, bytearray)): - output = item.decode() + output = item.decode("utf-8") elif isinstance(item, np.ndarray): if item.size == 0: output = item - elif "|S" in str(item.dtype) or isinstance(item[0], bytes): - output = [it.decode() for it in item] + elif "|S" in str(item.dtype) or (item.dtype.kind == 'S') or \ + (item.size > 0 and isinstance(item.flat[0], bytes)): + output = [it.decode("utf-8") for it in item] else: output = item - elif isinstance(item, np.bool_): + elif isinstance(item, (np.bool_, bool)): output = bool(item) else: output = item - return output + return output \ No newline at end of file From 5080b3b058a52ba62c4edd4e10a8717a280ebaad Mon Sep 17 00:00:00 2001 From: avi Date: Wed, 20 Aug 2025 16:57:19 +1200 Subject: [PATCH 07/11] upload image --- .github/workflows/compas-compile-ci.yml | 39 ++++++++++++++++++++----- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/.github/workflows/compas-compile-ci.yml b/.github/workflows/compas-compile-ci.yml index 5311d34a0..1692a0c4e 100644 --- a/.github/workflows/compas-compile-ci.yml +++ b/.github/workflows/compas-compile-ci.yml @@ -27,7 +27,6 @@ jobs: name: Build COMPAS runs-on: ubuntu-22.04 - # Use Docker Hub for now (GHCR is private) container: teamcompas/compas:latest # TODO: Switch to GHCR when package is made public # container: ghcr.io/teamcompas/compas:latest @@ -112,14 +111,40 @@ jobs: const workflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; let plotSection = ''; + try { if (fs.existsSync(evolutionPlotPath)) { - const imageData = fs.readFileSync(evolutionPlotPath, {encoding: 'base64'}); + const tag = `pr-${context.issue.number}-${Date.now()}`; + const plotData = fs.readFileSync(evolutionPlotPath); + + const release = await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: tag, + name: `Evolution Plot for PR #${context.issue.number}`, + body: `Temporary release to host evolution plot for PR #${context.issue.number}`, + draft: false, + prerelease: true + }); + + const asset = await github.rest.repos.uploadReleaseAsset({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: release.data.id, + name: 'detailedEvolutionPlot.png', + data: plotData, + headers: { + 'content-type': 'image/png' + } + }); + + const imageUrl = asset.data.browser_download_url; + plotSection = ` - ### 📊 Detailed Evolution Plot + ### Detailed Evolution Plot
Click to view evolution plot - ![Evolution Plot](data:image/png;base64,${imageData}) + ![Evolution Plot](${imageUrl})
`; } else { plotSection = '\n### ⚠️ Evolution plot not found'; @@ -127,7 +152,7 @@ jobs: } catch (error) { plotSection = `\n### ⚠️ Evolution plot error: ${error.message}`; } - + await github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, @@ -152,6 +177,4 @@ jobs: echo "- COMPAS Build: Success" >> $GITHUB_STEP_SUMMARY echo "- Python Dependencies: Success" >> $GITHUB_STEP_SUMMARY echo "- Example COMPAS Job: Success" >> $GITHUB_STEP_SUMMARY - echo "- Pytest Suite: Success" >> $GITHUB_STEP_SUMMARY - echo "- Container: teamcompas/compas:latest (Docker Hub)" >> $GITHUB_STEP_SUMMARY - echo "- Ready to switch to: ghcr.io/teamcompas/compas:latest" >> $GITHUB_STEP_SUMMARY \ No newline at end of file + echo "- Pytest Suite: Success" >> $GITHUB_STEP_SUMMARY \ No newline at end of file From a62c637ce60429e020ddf5be9f0977b9fdf5c3ed Mon Sep 17 00:00:00 2001 From: avi Date: Thu, 11 Sep 2025 12:14:00 +1200 Subject: [PATCH 08/11] fix image paths --- .github/workflows/compas-compile-ci.yml | 121 ++++----- .../workflows/pr_artifact_url_commenter.yml | 254 ------------------ 2 files changed, 47 insertions(+), 328 deletions(-) delete mode 100644 .github/workflows/pr_artifact_url_commenter.yml diff --git a/.github/workflows/compas-compile-ci.yml b/.github/workflows/compas-compile-ci.yml index 1692a0c4e..56130fd8f 100644 --- a/.github/workflows/compas-compile-ci.yml +++ b/.github/workflows/compas-compile-ci.yml @@ -95,86 +95,59 @@ jobs: id: upload uses: actions/upload-artifact@v4 with: - name: ${{ env.ARTIFACT_NAME }} + name: evolution-plot-${{ github.run_number }} path: ${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_NAME }} if-no-files-found: error - - name: Comment on PR with results - if: github.event_name == 'pull_request' - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const fs = require('fs'); - const evolutionPlotPath = '${{ env.ARTIFACT_PATH }}/${{ env.ARTIFACT_NAME }}'; - const shortSha = context.sha.substring(0, 7); - const workflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - - let plotSection = ''; - - try { - if (fs.existsSync(evolutionPlotPath)) { - const tag = `pr-${context.issue.number}-${Date.now()}`; - const plotData = fs.readFileSync(evolutionPlotPath); - - const release = await github.rest.repos.createRelease({ - owner: context.repo.owner, - repo: context.repo.repo, - tag_name: tag, - name: `Evolution Plot for PR #${context.issue.number}`, - body: `Temporary release to host evolution plot for PR #${context.issue.number}`, - draft: false, - prerelease: true - }); - - const asset = await github.rest.repos.uploadReleaseAsset({ - owner: context.repo.owner, - repo: context.repo.repo, - release_id: release.data.id, - name: 'detailedEvolutionPlot.png', - data: plotData, - headers: { - 'content-type': 'image/png' - } - }); - - const imageUrl = asset.data.browser_download_url; - - plotSection = ` - ### Detailed Evolution Plot -
Click to view evolution plot - - ![Evolution Plot](${imageUrl}) -
`; - } else { - plotSection = '\n### ⚠️ Evolution plot not found'; - } - } catch (error) { - plotSection = `\n### ⚠️ Evolution plot error: ${error.message}`; - } - - await github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `![badge](https://img.shields.io/badge/Build_Success-28a745?style=for-the-badge&logo=github-actions&logoColor=white) - - ## ✅ COMPAS Build Successful! - - | Item | Value | - |------|-------| - | **Commit** | [\`${shortSha}\`](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/commit/${context.sha}) | - | **Logs** | [View workflow](${workflowUrl}) | - ${plotSection} - - --- - Generated by COMPAS CI` - }); - - name: Test Summary run: | echo "### COMPAS CI Results" >> $GITHUB_STEP_SUMMARY echo "- COMPAS Build: Success" >> $GITHUB_STEP_SUMMARY echo "- Python Dependencies: Success" >> $GITHUB_STEP_SUMMARY echo "- Example COMPAS Job: Success" >> $GITHUB_STEP_SUMMARY - echo "- Pytest Suite: Success" >> $GITHUB_STEP_SUMMARY \ No newline at end of file + echo "- Pytest Suite: Success" >> $GITHUB_STEP_SUMMARY + + comment-with-plot: + name: Comment PR with Evolution Plot + runs-on: ubuntu-22.04 + container: docker://ghcr.io/iterative/cml:0-dvc2-base1 + needs: compas + if: github.event_name == 'pull_request' + + env: + ARTIFACT_NAME: detailedEvolutionPlot.png + + steps: + - name: Download evolution plot + uses: actions/download-artifact@v4 + with: + name: evolution-plot-${{ github.run_number }} + + - name: Create report with evolution plot + env: + REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "## ✅ COMPAS Build Successful!" >> report.md + echo "" >> report.md + echo "| Item | Value |" >> report.md + echo "|------|-------|" >> report.md + echo "| **Commit** | [\`${GITHUB_SHA:0:7}\`](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}) |" >> report.md + echo "| **Logs** | [View workflow](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) |" >> report.md + echo "" >> report.md + + if [ -f "${{ env.ARTIFACT_NAME }}" ]; then + echo "### 📊 Detailed Evolution Plot" >> report.md + echo "
Click to view evolution plot" >> report.md + echo "" >> report.md + echo "![](./${{ env.ARTIFACT_NAME }})" >> report.md + echo "
" >> report.md + else + echo "### ⚠️ Evolution plot not found" >> report.md + fi + + echo "" >> report.md + echo "---" >> report.md + echo "Generated by COMPAS CI with CML" >> report.md + + # Post the report using CML + cml comment create report.md \ No newline at end of file diff --git a/.github/workflows/pr_artifact_url_commenter.yml b/.github/workflows/pr_artifact_url_commenter.yml deleted file mode 100644 index 90a3604d3..000000000 --- a/.github/workflows/pr_artifact_url_commenter.yml +++ /dev/null @@ -1,254 +0,0 @@ -name: Comment on Pull request - -on: - workflow_run: - workflows: ['COMPAS compile test'] - types: - - completed - -jobs: - comment: - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - - name: Get Artifact and Pull request info - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WORKFLOW_RUN_EVENT_OBJ: ${{ toJSON(github.event.workflow_run) }} - OWNER: ${{ github.repository_owner }} - REPO: ${{ github.event.repository.name }} - run: | - # Extract workflow run info - PREVIOUS_JOB_ID=$(jq -r '.id' <<< "$WORKFLOW_RUN_EVENT_OBJ") - echo "PREVIOUS_JOB_ID=$PREVIOUS_JOB_ID" >> "$GITHUB_ENV" - - HEAD_SHA="${{ github.event.workflow_run.head_sha }}" - echo "HEAD_SHA=$HEAD_SHA" >> "$GITHUB_ENV" - - # Get PR number from the workflow run - echo "Looking for PR associated with commit $HEAD_SHA..." - PR_NUMBER=$(gh api "/repos/$OWNER/$REPO/commits/$HEAD_SHA/pulls" \ - --jq '.[0].number // empty' 2>/dev/null || echo "") - - if [[ -n "$PR_NUMBER" && "$PR_NUMBER" != "null" ]]; then - echo "Found PR #$PR_NUMBER" - echo "PR_NUMBER=$PR_NUMBER" >> "$GITHUB_ENV" - - # Look for the specific artifact - ARTIFACT_NAME="detailedEvolutionPlot.png" - echo "Searching for artifact: $ARTIFACT_NAME from workflow run $PREVIOUS_JOB_ID..." - - # Retry loop with better error handling - ARTIFACT_ID="" - for i in {1..8}; do - echo "Attempt $i/8: Looking for artifact..." - - # Use more robust jq filter - ARTIFACT_ID=$(gh api "/repos/$OWNER/$REPO/actions/artifacts" \ - --jq --arg workflow_id "$PREVIOUS_JOB_ID" --arg name "$ARTIFACT_NAME" \ - '.artifacts[] | select(.workflow_run.id == ($workflow_id | tonumber) and .expired == false and .name == $name) | .id' \ - 2>/dev/null | head -n1 || echo "") - - if [[ -n "$ARTIFACT_ID" ]]; then - echo "Found artifact with ID: $ARTIFACT_ID" - break - fi - - echo "Artifact not available yet, waiting 12s before retry..." - sleep 12 - done - - if [[ -z "$ARTIFACT_ID" ]]; then - echo "Warning: Could not find artifact '$ARTIFACT_NAME' after 8 attempts" - echo "This might be expected if the artifact generation failed or is still processing." - fi - - echo "ARTIFACT_ID=${ARTIFACT_ID:-}" >> "$GITHUB_ENV" - else - echo "No PR found for this workflow run (this is normal for direct pushes to main/dev)" - echo "PR_NUMBER=" >> "$GITHUB_ENV" - echo "ARTIFACT_ID=" >> "$GITHUB_ENV" - fi - - - name: Download artifact - if: env.PR_NUMBER != '' && env.ARTIFACT_ID != '' - run: | - echo "Downloading artifact ${{ env.ARTIFACT_ID }}..." - - # Download with better error handling - if gh api "/repos/${{ github.repository }}/actions/artifacts/${{ env.ARTIFACT_ID }}/zip" --output artifact.zip; then - echo "Successfully downloaded artifact.zip" - - # Extract and verify - if unzip -q artifact.zip; then - rm artifact.zip - - if [[ -f "detailedEvolutionPlot.png" ]]; then - echo "Found detailedEvolutionPlot.png" - echo "File size: $(du -h detailedEvolutionPlot.png | cut -f1)" - # Verify it's actually a PNG file - if file detailedEvolutionPlot.png | grep -q "PNG image"; then - echo "Verified as valid PNG image" - else - echo "Warning: File may not be a valid PNG" - fi - else - echo "Error: detailedEvolutionPlot.png not found in extracted files" - echo "Available files:" - ls -la - exit 1 - fi - else - echo "Error: Failed to extract artifact.zip" - exit 1 - fi - else - echo "Error: Failed to download artifact" - exit 1 - fi - - - name: Create success comment with artifact - if: env.PR_NUMBER != '' && env.ARTIFACT_ID != '' - env: - JOB_PATH: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ env.PREVIOUS_JOB_ID }}" - ARTIFACT_DL: "${{ github.server_url }}/${{ github.repository }}/suites/${{ github.event.workflow_run.check_suite_id }}/artifacts/${{ env.ARTIFACT_ID }}" - uses: actions/github-script@v7 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - const fs = require('fs'); - - try { - // Read and validate the image - if (!fs.existsSync('detailedEvolutionPlot.png')) { - throw new Error('detailedEvolutionPlot.png file not found'); - } - - const stats = fs.statSync('detailedEvolutionPlot.png'); - console.log(`Image file size: ${(stats.size / 1024).toFixed(1)} KB`); - - if (stats.size === 0) { - throw new Error('detailedEvolutionPlot.png is empty'); - } - - // GitHub has a ~65KB limit for image embeds, warn if approaching - if (stats.size > 50000) { - console.log('Warning: Image is quite large, may not display properly in GitHub'); - } - - const imageData = fs.readFileSync('detailedEvolutionPlot.png', {encoding: 'base64'}); - const shortSha = process.env.HEAD_SHA.substring(0, 7); - - const commentBody = \`![badge] - - ## Build Successful! - - Your COMPAS compilation and test suite completed successfully! - - | Item | Value | - |------|-------| - | **Commit** | [\\\`\${shortSha}\\\`](https://github.com/\${context.repo.owner}/\${context.repo.repo}/commit/\${process.env.HEAD_SHA}) | - | **Logs** | [View workflow logs](\${process.env.JOB_PATH}) | - | **Artifact** | [Download evolution plot](\${process.env.ARTIFACT_DL}) | - - ### Detailed Evolution Plot - -
- Click to view/hide evolution plot - - ![Detailed Evolution Plot](data:image/png;base64,\${imageData}) - -
- - --- - Generated by COMPAS CI • [View workflow run](\${process.env.JOB_PATH}) - - [badge]: https://img.shields.io/badge/Build_Success-28a745?style=for-the-badge&logo=github-actions&logoColor=white\`; - - await github.rest.issues.createComment({ - issue_number: parseInt(process.env.PR_NUMBER), - owner: context.repo.owner, - repo: context.repo.repo, - body: commentBody - }); - - console.log(`Successfully commented on PR #${process.env.PR_NUMBER} with artifact`); - - } catch (error) { - console.error('Error creating comment with artifact:', error); - - // Fallback: create comment without image - const fallbackBody = \`![badge] - - ## Build Successful! - - Your COMPAS compilation completed successfully, but there was an issue displaying the evolution plot. - - | Item | Value | - |------|-------| - | **Commit** | [\\\`\${process.env.HEAD_SHA.substring(0, 7)}\\\`](https://github.com/\${context.repo.owner}/\${context.repo.repo}/commit/\${process.env.HEAD_SHA}) | - | **Logs** | [View workflow logs](\${process.env.JOB_PATH}) | - | **Artifact** | [Download evolution plot](\${process.env.ARTIFACT_DL}) | - - **Note:** Evolution plot could not be embedded (\${error.message}) - - --- - Generated by COMPAS CI • [View workflow run](\${process.env.JOB_PATH}) - - [badge]: https://img.shields.io/badge/Build_Success-28a745?style=for-the-badge&logo=github-actions&logoColor=white\`; - - await github.rest.issues.createComment({ - issue_number: parseInt(process.env.PR_NUMBER), - owner: context.repo.owner, - repo: context.repo.repo, - body: fallbackBody - }); - - console.log(`Created fallback comment on PR #${process.env.PR_NUMBER}`); - } - - - name: Create success comment without artifact - if: env.PR_NUMBER != '' && env.ARTIFACT_ID == '' - env: - JOB_PATH: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ env.PREVIOUS_JOB_ID }}" - uses: actions/github-script@v7 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - const shortSha = process.env.HEAD_SHA.substring(0, 7); - - const commentBody = \`![badge] - - ## Build Successful! - - Your COMPAS compilation and test suite completed successfully! - - | Item | Value | - |------|-------| - | **Commit** | [\\\`\${shortSha}\\\`](https://github.com/\${context.repo.owner}/\${context.repo.repo}/commit/\${process.env.HEAD_SHA}) | - | **Logs** | [View workflow logs](\${process.env.JOB_PATH}) | - - **Note:** The detailed evolution plot artifact was not found or is not yet available. This might be expected if the plot generation step was skipped or failed. - - --- - Generated by COMPAS CI • [View workflow run](\${process.env.JOB_PATH}) - - [badge]: https://img.shields.io/badge/Build_Success-28a745?style=for-the-badge&logo=github-actions&logoColor=white\`; - - await github.rest.issues.createComment({ - issue_number: parseInt(process.env.PR_NUMBER), - owner: context.repo.owner, - repo: context.repo.repo, - body: commentBody - }); - - console.log(`Successfully commented on PR #${process.env.PR_NUMBER} (no artifact)`); - - - name: Debug info for non-PR runs - if: env.PR_NUMBER == '' - run: | - echo "This workflow run was not associated with a pull request" - echo "Head SHA: ${{ github.event.workflow_run.head_sha }}" - echo "Event: ${{ github.event.workflow_run.event }}" - echo "Branch: ${{ github.event.workflow_run.head_branch }}" - echo "This is normal for direct pushes to main/dev branches" \ No newline at end of file From 43f2c6cc5d4b773e45d471c9df1d77baf9b9bc27 Mon Sep 17 00:00:00 2001 From: avi Date: Thu, 11 Sep 2025 12:23:27 +1200 Subject: [PATCH 09/11] fix bad SHA --- .github/workflows/compas-compile-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compas-compile-ci.yml b/.github/workflows/compas-compile-ci.yml index 56130fd8f..7341b07b0 100644 --- a/.github/workflows/compas-compile-ci.yml +++ b/.github/workflows/compas-compile-ci.yml @@ -131,7 +131,7 @@ jobs: echo "" >> report.md echo "| Item | Value |" >> report.md echo "|------|-------|" >> report.md - echo "| **Commit** | [\`${GITHUB_SHA:0:7}\`](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}) |" >> report.md + echo "| **Commit** | [\`$(echo $GITHUB_SHA | cut -c1-7)\`](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}) |" >> report.md echo "| **Logs** | [View workflow](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) |" >> report.md echo "" >> report.md From 524353023ff60aa7fa8ff1dd81609d43c19ddc8b Mon Sep 17 00:00:00 2001 From: avi Date: Thu, 11 Sep 2025 12:34:23 +1200 Subject: [PATCH 10/11] need to checkout at start of the CI --- .github/workflows/compas-compile-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/compas-compile-ci.yml b/.github/workflows/compas-compile-ci.yml index 7341b07b0..3d3aefe4c 100644 --- a/.github/workflows/compas-compile-ci.yml +++ b/.github/workflows/compas-compile-ci.yml @@ -118,6 +118,9 @@ jobs: ARTIFACT_NAME: detailedEvolutionPlot.png steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Download evolution plot uses: actions/download-artifact@v4 with: From 5b0f9b518219e992d4052653a2814c8b3631d14d Mon Sep 17 00:00:00 2001 From: avi Date: Thu, 11 Sep 2025 12:51:47 +1200 Subject: [PATCH 11/11] edit comment --- .github/workflows/compas-compile-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compas-compile-ci.yml b/.github/workflows/compas-compile-ci.yml index 3d3aefe4c..4ee1eb40d 100644 --- a/.github/workflows/compas-compile-ci.yml +++ b/.github/workflows/compas-compile-ci.yml @@ -139,7 +139,7 @@ jobs: echo "" >> report.md if [ -f "${{ env.ARTIFACT_NAME }}" ]; then - echo "### 📊 Detailed Evolution Plot" >> report.md + echo "### Detailed Evolution Plot" >> report.md echo "
Click to view evolution plot" >> report.md echo "" >> report.md echo "![](./${{ env.ARTIFACT_NAME }})" >> report.md @@ -150,7 +150,7 @@ jobs: echo "" >> report.md echo "---" >> report.md - echo "Generated by COMPAS CI with CML" >> report.md + echo "Generated by COMPAS CI " >> report.md # Post the report using CML cml comment create report.md \ No newline at end of file