Skip to content

Commit

Permalink
ci: update gas benchmark CI to display gas diffs as well as the figur…
Browse files Browse the repository at this point in the history
…es (#731)

* build: write Hardhat task for gas benchmark

* refactor: improve benchmark generation script with markdown

* ci: update CI to generate gas comparison

* build: add deployment cost in gas benchmark
  • Loading branch information
CJ42 authored Sep 29, 2023
1 parent b846c77 commit d62239e
Show file tree
Hide file tree
Showing 6 changed files with 1,143 additions and 358 deletions.
43 changes: 39 additions & 4 deletions .github/workflows/benchmark.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
# This workflow benchmark the gas usage of Universal Profile for common interactions

# It compare the gas cost of the changes made between:
# - a feature branch (where a PR is opened)
# - a target branch (where the PR will be merged)
name: 🆙 📊 Universal Profile Benchmark

on:
Expand All @@ -22,7 +24,11 @@ jobs:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3
- name: Checkout base branch
uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.base.sha }}
fetch-depth: 0

- name: Use Node.js '16.15.0'
uses: actions/setup-node@v2
Expand All @@ -37,11 +43,40 @@ jobs:
run: npx hardhat compile

- name: 🧪 Run Benchmark tests
run: npm run test:benchmark
# Rename the file to be able to generate benchmark JSON report
run: |
npm run test:benchmark
mv gas_benchmark_result.json gas_benchmark_before.json
- name: Checkout current branch
uses: actions/checkout@v3
# Do not run `git clean -ffdx && git reset --hard HEAD` to prevent removing `gas_benchmark_before.json`
with:
clean: false

- name: Use Node.js '16.15.0'
uses: actions/setup-node@v2
with:
node-version: "16.15.0"
cache: "npm"

- name: 📦 Install dependencies
run: npm ci

- name: 🏗️ Build contract artifacts
run: npx hardhat compile

- name: 🧪 Run Benchmark tests
run: |
npm run test:benchmark
mv gas_benchmark_result.json gas_benchmark_after.json
- name: 📊 Generate Benchmark Report
run: npx hardhat gas-benchmark --compare gas_benchmark_after.json --against gas_benchmark_before.json

- name: 💬 Add Gas Report as comment in PR
uses: peter-evans/create-or-update-comment@v2
with:
token: ${{ secrets.GITHUB_TOKEN }}
issue-number: ${{ github.event.pull_request.number }}
body-file: "./benchmark.md"
body-file: "./gas_benchmark.md"
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ out/
forge-cache/

# generated gas benchmark
benchmark.md
gas_benchmark.md

# Exclude build output folders
/common
Expand Down
3 changes: 3 additions & 0 deletions hardhat.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,10 @@ import '@nomicfoundation/hardhat-toolbox';
import 'hardhat-packager';
import 'hardhat-contract-sizer';
import 'hardhat-deploy';

// custom built hardhat plugins for CI
import './scripts/ci/docs-generate';
import './scripts/ci/gas_benchmark';

// Typescript types for web3.js
import '@nomiclabs/hardhat-web3';
Expand Down
226 changes: 226 additions & 0 deletions scripts/ci/gas_benchmark.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,226 @@
import fs from 'fs';
import { task } from 'hardhat/config';
import { Align, getMarkdownTable, Row } from 'markdown-table-ts';

task('gas-benchmark', 'Benchmark gas usage of the smart contracts based on predefined scenarios')
.addParam(
'compare',
'The `.json` file that contains the gas costs of the currently compiled contracts (e.g: current working branch)',
)
.addParam(
'against',
'The `.json` file that contains the gas costs to compare against (e.g: the `develop` branch)',
)
.setAction(async function (args) {
const currentBenchmark = JSON.parse(fs.readFileSync(args.compare, 'utf8'));
const baseBenchmark = JSON.parse(fs.readFileSync(args.against, 'utf8'));

const deploymentCosts: Row[] = [];

const casesEOAExecute: Row[] = [];
const casesEOASetData: Row[] = [];
const casesEOATokens: Row[] = [];

const casesKeyManagerExecute: Row[] = [];
const casesKeyManagerSetData: Row[] = [];

const formatNumber = (value: number) => {
return value.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ',');
};

const displayGasDiff = (gasDiff: number) => {
let emoji = '';

if (gasDiff > 0) {
emoji = '📈❌';
}

if (gasDiff < 0) {
emoji = '📉✅';
}

return `${formatNumber(gasDiff)} ${emoji}`;
};

// Deployment costs
for (const [key, value] of Object.entries(currentBenchmark['deployment_costs'])) {
const gasCost: any = value;
const gasDiff = gasCost - baseBenchmark['deployment_costs'][key];

deploymentCosts.push([key, value + ` (${displayGasDiff(gasDiff)})`]);
}

const generatedDeploymentCostsTable = getMarkdownTable({
table: {
head: ['Deployed contracts', '⛽ Deployment cost'],
body: deploymentCosts,
},
alignment: [Align.Left],
});

// EOA - execute
for (const [key, value] of Object.entries(
currentBenchmark['runtime_costs']['EOA_owner']['execute'],
)) {
const gasDiff =
value['gas_cost'] - baseBenchmark['runtime_costs']['EOA_owner']['execute'][key]['gas_cost'];

casesEOAExecute.push([
value['description'],
value['gas_cost'] + ` (${displayGasDiff(gasDiff)})`,
]);
}

const generatedEOAExecuteTable = getMarkdownTable({
table: {
head: ['`execute` scenarios - UP owned by 🔑 EOA', '⛽ Gas Usage'],
body: casesEOAExecute,
},
alignment: [Align.Left],
});

// EOA - setData
for (const [key, value] of Object.entries(
currentBenchmark['runtime_costs']['EOA_owner']['setData'],
)) {
const gasDiff =
value['gas_cost'] - baseBenchmark['runtime_costs']['EOA_owner']['setData'][key]['gas_cost'];

casesEOASetData.push([
value['description'],
value['gas_cost'] + ` (${displayGasDiff(gasDiff)})`,
]);
}

const generatedEOASetDataTable = getMarkdownTable({
table: {
head: ['`setData` scenarios - UP owned by 🔑 EOA', '⛽ Gas Usage'],
body: casesEOASetData,
},
alignment: [Align.Left],
});

// EOA - Tokens
for (const [key, value] of Object.entries(
currentBenchmark['runtime_costs']['EOA_owner']['tokens'],
)) {
const gasDiff =
value['gas_cost'] - baseBenchmark['runtime_costs']['EOA_owner']['tokens'][key]['gas_cost'];

casesEOATokens.push([
value['description'],
value['gas_cost'] + ` (${displayGasDiff(gasDiff)})`,
]);
}

const generatedEOATokensTable = getMarkdownTable({
table: {
head: ['`Tokens` scenarios - UP owned by 🔑 EOA', '⛽ Gas Usage'],
body: casesEOATokens,
},
alignment: [Align.Left],
});

// Key Manager - execute
for (const [key, value] of Object.entries(
currentBenchmark['runtime_costs']['KeyManager_owner']['execute'],
)) {
const gasDiffMainController =
value['main_controller'] -
baseBenchmark['runtime_costs']['KeyManager_owner']['execute'][key]['main_controller'];

const gasDiffRestrictedController =
value['restricted_controller'] -
baseBenchmark['runtime_costs']['KeyManager_owner']['execute'][key]['restricted_controller'];

casesKeyManagerExecute.push([
value['description'],
value['main_controller'] + ` (${displayGasDiff(gasDiffMainController)})`,
value['restricted_controller'] + ` (${displayGasDiff(gasDiffRestrictedController)})`,
]);
}

const generatedKeyManagerExecuteTable = getMarkdownTable({
table: {
head: ['`execute` scenarios', '👑 main controller', '🛃 restricted controller'],
body: casesKeyManagerExecute,
},
alignment: [Align.Left],
});

// Key Manager - setData
for (const [key, value] of Object.entries(
currentBenchmark['runtime_costs']['KeyManager_owner']['setData'],
)) {
const gasDiffMainController =
value['main_controller'] -
baseBenchmark['runtime_costs']['KeyManager_owner']['setData'][key]['main_controller'];

const gasDiffRestrictedController =
value['restricted_controller'] -
baseBenchmark['runtime_costs']['KeyManager_owner']['setData'][key]['restricted_controller'];

casesKeyManagerSetData.push([
value['description'],
value['main_controller'] + ` (${displayGasDiff(gasDiffMainController)})`,
value['restricted_controller'] + ` (${displayGasDiff(gasDiffRestrictedController)})`,
]);
}

const generatedKeyManagerSetDataTable = getMarkdownTable({
table: {
head: ['`setData` scenarios', '👑 main controller', '🛃 restricted controller'],
body: casesKeyManagerSetData,
},
alignment: [Align.Left],
});

const markdownContent = `
👋 Hello
⛽ I am the Gas Bot Reporter. I keep track of the gas costs of common interactions using Universal Profiles 🆙 !
📊 Here is a summary of the gas cost with the code introduced by this PR.
## ⛽📊 Gas Benchmark Report
### Deployment Costs
${generatedDeploymentCostsTable}
### Runtime Costs
<details>
<summary>UniversalProfile owned by an 🔑 EOA</summary>
### 🔀 \`execute\` scenarios
${generatedEOAExecuteTable}
### 🗄️ \`setData\` scenarios
${generatedEOASetDataTable}
### 🗄️ \`Tokens\` scenarios
${generatedEOATokensTable}
</details>
<details>
<summary>UniversalProfile owned by a 🔒📄 LSP6KeyManager</summary>
### 🔀 \`execute\` scenarios
${generatedKeyManagerExecuteTable}
### 🗄️ \`setData\` scenarios
${generatedKeyManagerSetDataTable}
</details>
`;

const file = 'gas_benchmark.md';

fs.writeFileSync(file, markdownContent, 'utf8');
});
Loading

0 comments on commit d62239e

Please sign in to comment.