ci: update gas benchmark CI to display gas diffs as well as the figures

.github/workflows/benchmark.yml

@@ -1,5 +1,7 @@
 # This workflow benchmark the gas usage of Universal Profile for common interactions
-
+# It compare the gas cost of the changes made between:
+# - a feature branch (where a PR is opened)
+# - a target branch (where the PR will be merged)
 name: 🆙 📊 Universal Profile Benchmark
 
 on:
@@ -22,7 +24,11 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout base branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          fetch-depth: 0
 
       - name: Use Node.js '16.15.0'
         uses: actions/setup-node@v2
@@ -37,11 +43,40 @@ jobs:
         run: npx hardhat compile
 
       - name: 🧪 Run Benchmark tests
-        run: npm run test:benchmark
+        # Rename the file to be able to generate benchmark JSON report
+        run: |
+          npm run test:benchmark
+          mv gas_benchmark_result.json gas_benchmark_before.json
+
+      - name: Checkout current branch
+        uses: actions/checkout@v3
+        # Do not run `git clean -ffdx && git reset --hard HEAD` to prevent removing `gas_benchmark_before.json`
+        with:
+          clean: false
+
+      - name: Use Node.js '16.15.0'
+        uses: actions/setup-node@v2
+        with:
+          node-version: "16.15.0"
+          cache: "npm"
+
+      - name: 📦 Install dependencies
+        run: npm ci
+
+      - name: 🏗️ Build contract artifacts
+        run: npx hardhat compile
+
+      - name: 🧪 Run Benchmark tests
+        run: |
+          npm run test:benchmark
+          mv gas_benchmark_result.json gas_benchmark_after.json
 
       - name: 📊 Generate Benchmark Report
+        run: npx hardhat gas-benchmark --compare gas_benchmark_after.json --against gas_benchmark_before.json
+
+      - name: 💬 Add Gas Report as comment in PR
         uses: peter-evans/create-or-update-comment@v2
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           issue-number: ${{ github.event.pull_request.number }}
-          body-file: "./benchmark.md"
+          body-file: "./gas_benchmark.md"

.gitignore

@@ -132,7 +132,7 @@ out/
 forge-cache/
 
 # generated gas benchmark
-benchmark.md
+gas_benchmark.md
 
 # Exclude build output folders
 /common

hardhat.config.ts

@@ -18,7 +18,10 @@ import '@nomicfoundation/hardhat-toolbox';
 import 'hardhat-packager';
 import 'hardhat-contract-sizer';
 import 'hardhat-deploy';
+
+// custom built hardhat plugins for CI
 import './scripts/ci/docs-generate';
+import './scripts/ci/gas_benchmark';
 
 // Typescript types for web3.js
 import '@nomiclabs/hardhat-web3';

scripts/ci/gas_benchmark.ts

@@ -0,0 +1,226 @@
+import fs from 'fs';
+import { task } from 'hardhat/config';
+import { Align, getMarkdownTable, Row } from 'markdown-table-ts';
+
+task('gas-benchmark', 'Benchmark gas usage of the smart contracts based on predefined scenarios')
+  .addParam(
+    'compare',
+    'The `.json` file that contains the gas costs of the currently compiled contracts (e.g: current working branch)',
+  )
+  .addParam(
+    'against',
+    'The `.json` file that contains the gas costs to compare against (e.g: the `develop` branch)',
+  )
+  .setAction(async function (args) {
+    const currentBenchmark = JSON.parse(fs.readFileSync(args.compare, 'utf8'));
+    const baseBenchmark = JSON.parse(fs.readFileSync(args.against, 'utf8'));
+
+    const deploymentCosts: Row[] = [];
+
+    const casesEOAExecute: Row[] = [];
+    const casesEOASetData: Row[] = [];
+    const casesEOATokens: Row[] = [];
+
+    const casesKeyManagerExecute: Row[] = [];
+    const casesKeyManagerSetData: Row[] = [];
+
+    const formatNumber = (value: number) => {
+      return value.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ',');
+    };
+
+    const displayGasDiff = (gasDiff: number) => {
+      let emoji = '';
+
+      if (gasDiff > 0) {
+        emoji = '📈❌';
+      }
+
+      if (gasDiff < 0) {
+        emoji = '📉✅';
+      }
+
+      return `${formatNumber(gasDiff)} ${emoji}`;
+    };
+
+    // Deployment costs
+    for (const [key, value] of Object.entries(currentBenchmark['deployment_costs'])) {
+      const gasCost: any = value;
+      const gasDiff = gasCost - baseBenchmark['deployment_costs'][key];
+
+      deploymentCosts.push([key, value + ` (${displayGasDiff(gasDiff)})`]);
+    }
+
+    const generatedDeploymentCostsTable = getMarkdownTable({
+      table: {
+        head: ['Deployed contracts', '⛽ Deployment cost'],
+        body: deploymentCosts,
+      },
+      alignment: [Align.Left],
+    });
+
+    // EOA - execute
+    for (const [key, value] of Object.entries(
+      currentBenchmark['runtime_costs']['EOA_owner']['execute'],
+    )) {
+      const gasDiff =
+        value['gas_cost'] - baseBenchmark['runtime_costs']['EOA_owner']['execute'][key]['gas_cost'];
+
+      casesEOAExecute.push([
+        value['description'],
+        value['gas_cost'] + ` (${displayGasDiff(gasDiff)})`,
+      ]);
+    }
+
+    const generatedEOAExecuteTable = getMarkdownTable({
+      table: {
+        head: ['`execute` scenarios - UP owned by 🔑 EOA', '⛽ Gas Usage'],
+        body: casesEOAExecute,
+      },
+      alignment: [Align.Left],
+    });
+
+    // EOA - setData
+    for (const [key, value] of Object.entries(
+      currentBenchmark['runtime_costs']['EOA_owner']['setData'],
+    )) {
+      const gasDiff =
+        value['gas_cost'] - baseBenchmark['runtime_costs']['EOA_owner']['setData'][key]['gas_cost'];
+
+      casesEOASetData.push([
+        value['description'],
+        value['gas_cost'] + ` (${displayGasDiff(gasDiff)})`,
+      ]);
+    }
+
+    const generatedEOASetDataTable = getMarkdownTable({
+      table: {
+        head: ['`setData` scenarios - UP owned by 🔑 EOA', '⛽ Gas Usage'],
+        body: casesEOASetData,
+      },
+      alignment: [Align.Left],
+    });
+
+    // EOA - Tokens
+    for (const [key, value] of Object.entries(
+      currentBenchmark['runtime_costs']['EOA_owner']['tokens'],
+    )) {
+      const gasDiff =
+        value['gas_cost'] - baseBenchmark['runtime_costs']['EOA_owner']['tokens'][key]['gas_cost'];
+
+      casesEOATokens.push([
+        value['description'],
+        value['gas_cost'] + ` (${displayGasDiff(gasDiff)})`,
+      ]);
+    }
+
+    const generatedEOATokensTable = getMarkdownTable({
+      table: {
+        head: ['`Tokens` scenarios - UP owned by 🔑 EOA', '⛽ Gas Usage'],
+        body: casesEOATokens,
+      },
+      alignment: [Align.Left],
+    });
+
+    // Key Manager - execute
+    for (const [key, value] of Object.entries(
+      currentBenchmark['runtime_costs']['KeyManager_owner']['execute'],
+    )) {
+      const gasDiffMainController =
+        value['main_controller'] -
+        baseBenchmark['runtime_costs']['KeyManager_owner']['execute'][key]['main_controller'];
+
+      const gasDiffRestrictedController =
+        value['restricted_controller'] -
+        baseBenchmark['runtime_costs']['KeyManager_owner']['execute'][key]['restricted_controller'];
+
+      casesKeyManagerExecute.push([
+        value['description'],
+        value['main_controller'] + ` (${displayGasDiff(gasDiffMainController)})`,
+        value['restricted_controller'] + ` (${displayGasDiff(gasDiffRestrictedController)})`,
+      ]);
+    }
+
+    const generatedKeyManagerExecuteTable = getMarkdownTable({
+      table: {
+        head: ['`execute` scenarios', '👑 main controller', '🛃 restricted controller'],
+        body: casesKeyManagerExecute,
+      },
+      alignment: [Align.Left],
+    });
+
+    // Key Manager - setData
+    for (const [key, value] of Object.entries(
+      currentBenchmark['runtime_costs']['KeyManager_owner']['setData'],
+    )) {
+      const gasDiffMainController =
+        value['main_controller'] -
+        baseBenchmark['runtime_costs']['KeyManager_owner']['setData'][key]['main_controller'];
+
+      const gasDiffRestrictedController =
+        value['restricted_controller'] -
+        baseBenchmark['runtime_costs']['KeyManager_owner']['setData'][key]['restricted_controller'];
+
+      casesKeyManagerSetData.push([
+        value['description'],
+        value['main_controller'] + ` (${displayGasDiff(gasDiffMainController)})`,
+        value['restricted_controller'] + ` (${displayGasDiff(gasDiffRestrictedController)})`,
+      ]);
+    }
+
+    const generatedKeyManagerSetDataTable = getMarkdownTable({
+      table: {
+        head: ['`setData` scenarios', '👑 main controller', '🛃 restricted controller'],
+        body: casesKeyManagerSetData,
+      },
+      alignment: [Align.Left],
+    });
+
+    const markdownContent = `
+👋 Hello
+⛽ I am the Gas Bot Reporter. I keep track of the gas costs of common interactions using Universal Profiles 🆙 !
+📊 Here is a summary of the gas cost with the code introduced by this PR.
+
+## ⛽📊 Gas Benchmark Report
+
+### Deployment Costs
+
+${generatedDeploymentCostsTable}
+
+### Runtime Costs
+
+<details>
+<summary>UniversalProfile owned by an 🔑 EOA</summary>
+
+### 🔀 \`execute\` scenarios
+
+${generatedEOAExecuteTable}
+
+### 🗄️ \`setData\` scenarios
+
+${generatedEOASetDataTable}
+
+### 🗄️ \`Tokens\` scenarios
+
+${generatedEOATokensTable}
+
+</details>
+
+<details>
+<summary>UniversalProfile owned by a 🔒📄 LSP6KeyManager</summary>
+
+### 🔀 \`execute\` scenarios
+
+${generatedKeyManagerExecuteTable}
+
+### 🗄️ \`setData\` scenarios
+
+${generatedKeyManagerSetDataTable}
+
+</details>
+
+    `;
+
+    const file = 'gas_benchmark.md';
+
+    fs.writeFileSync(file, markdownContent, 'utf8');
+  });