diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000000..941627c49a --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,143 @@ +name: Benchmark + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + +permissions: + contents: read + pull-requests: write + +jobs: + benchmarks: + if: contains(github.event.pull_request.labels.*.name, 'request-benchmarks') + runs-on: ubuntu-latest + + steps: + - name: Install wrk + run: | + sudo apt-get update + sudo apt-get install -y wrk + + - name: Checkout Source Branch + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.ref }} + path: source + + - name: Setup Node.js for Source Branch + uses: actions/setup-node@v3 + with: + node-version: '22.x' + + - name: Install Dependencies for Source Branch + run: | + cd source + npm install + + - name: Run Benchmarks on Source Branch + run: | + cd source/benchmarks + make > source_results.log + + - name: Save Source Results + id: save-source-results + uses: actions/upload-artifact@v2 + with: + name: source-results + path: source/benchmarks/source_results.log + + - name: Checkout Target Branch + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.base.ref }} + path: target + + - name: Setup Node.js for Target Branch + uses: actions/setup-node@v3 + with: + node-version: '22.x' + + - name: Install Dependencies for Target Branch + run: | + cd target + npm install + + - name: Run Benchmarks on Target Branch + run: | + cd target/benchmarks + make > target_results.log + + - name: Save Target Results + id: save-target-results + uses: actions/upload-artifact@v2 + with: + name: target-results + path: target/benchmarks/target_results.log + + - name: Download Source Results + uses: actions/download-artifact@v2 + with: + name: source-results + path: source-results + + - name: Download Target Results + uses: actions/download-artifact@v2 + with: + name: target-results + path: target-results + + - name: Compare Results and Comment on PR + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_HEAD_REF: ${{ github.event.pull_request.head.ref }} + PR_BASE_REF: ${{ github.event.pull_request.base.ref }} + PR_COMMENTS_URL: ${{ github.event.pull_request.comments_url }} + run: | + set -e + echo "Comparing benchmark results..." + # Check if source results file exists + if [ ! -f source-results/source_results.log ]; then + echo "Source results file not found!" + exit 1 + fi + + # Check if target results file exists + if [ ! -f target-results/target_results.log ]; then + echo "Target results file not found!" + exit 1 + fi + + # Compare the benchmark results and generate a summary + SOURCE_RESULTS=$(cat source-results/source_results.log) + TARGET_RESULTS=$(cat target-results/target_results.log) + DIFF=$(diff <(echo "$SOURCE_RESULTS") <(echo "$TARGET_RESULTS")) || true + + # Debugging output + echo "Source Results:" + echo "$SOURCE_RESULTS" + echo "Target Results:" + echo "$TARGET_RESULTS" + echo "Differences:" + echo "$DIFF" + + # Create a comment body + COMMENT_BODY='### Benchmark Comparison\n\n' + COMMENT_BODY+="**Source Branch (${PR_HEAD_REF}) Results:**\n\`\`\`\n${SOURCE_RESULTS}\n\`\`\`\n\n" + COMMENT_BODY+="**Target Branch (${PR_BASE_REF}) Results:**\n\`\`\`\n${TARGET_RESULTS}\n\`\`\`\n\n" + COMMENT_BODY+="**Differences:**\n\`\`\`\n${DIFF}\n\`\`\`\n" + + # Post the comment to the PR + PAYLOAD=$(jq -n --arg body "$COMMENT_BODY" '{body: $body}') + curl -s -X POST -H "Authorization: token $GITHUB_TOKEN" \ + -H "Content-Type: application/json" \ + -d "$PAYLOAD" \ + "$PR_COMMENTS_URL" + continue-on-error: true + + - name: Remove request-benchmarks Label + run: | + curl -s -X DELETE -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Content-Type: application/json" \ + "${{ github.event.pull_request.issue_url }}/labels/request-benchmarks"