🧐 ERC implementation benchmarks

cairoeth · Jan 9, 2024 · d77763a · d77763a
1 parent b526520
commit d77763a
Show file tree

Hide file tree

Showing 10 changed files with 247 additions and 129 deletions.
diff --git a/README.md b/README.md
@@ -47,24 +47,71 @@ Import the contracts you want to use:
 use rustmate::tokens::erc20::{ERC20, ERC20Params};
 ```
 
-## ✅ Benchmarks
+## ✅ Gas benchmarks
 
 ### 🧪 Results
 
 <details><summary>ERC20</summary>
 
-| Function | Rustmate | Solmate | OpenZeppelin Contracts
-| -------- | -------- | -------- | -------- |
-| name()   | TBD    | TBD   | TBD    |
-| symbol()   | TBD   | TBD   | TBD    |
-| decimals()   | TBD   | TBD   | TBD    |
-| totalSupply()   | TBD   | TBD   | TBD    |
-| balanceOf(address)   | TBD   | TBD   | TBD    |
-| allowance(address,address)   | TBD   | TBD   | TBD    |
-| nonces(address)   | TBD   | TBD   | TBD    |
-| approve(address,uint256)   | TBD   | TBD   | TBD    |
-| transfer(address,uint256)   | TBD   | TBD   | TBD    |
-| transferFrom(address,address,uint256)   | TBD   | TBD   | TBD    |
+|    Function    | Rustmate | Solmate | OpenZeppelin Contracts 5.0 |
+|:--------------:|:--------:|:-------:|:--------------------------:|
+|     name()     |   23043  |  24504  |            24514           |
+|    symbol()    |   22974  |  24571  |            24535           |
+|   decimals()   |   22726  |  21512  |            21520           |
+|  totalSupply() |   25617  |  23562  |            23570           |
+|   balanceOf()  |   26851  |  24292  |            24296           |
+|   allowance()  |   28263  |  25011  |            25066           |
+|    nonces()    |   26835  |  24302  |             N/A            |
+|    approve()   |   50557  |  46683  |            46902           |
+|   transfer()   |   74234  |  47133  |            27454           |
+| transferFrom() |   60116  |  28993  |            29202           |
+
+</details>
+
+<details><summary>ERC721</summary>
+
+|       Function      | Rustmate | Solmate | OpenZeppelin Contracts 5.0 |
+|:-------------------:|:--------:|:-------:|:--------------------------:|
+|        name()       |   23286  |  24548  |            24556           |
+|       symbol()      |   23225  |  24548  |            24556           |
+|      ownerOf()      |   24212  |  24212  |            24308           |
+|     balanceOf()     |   27094  |  24352  |            24352           |
+|    getApproved()    |   26749  |  24132  |            26545           |
+|  isApprovedForAll() |   28447  |  25046  |            25104           |
+|      tokenURI()     |   24293  |  23420  |            23420           |
+|      approve()      |   48639  |  48693  |            49043           |
+| setApprovalForAll() |   51279  |  46561  |            46669           |
+|    transferFrom()   |   32777  |  32437  |            32947           |
+|  safeTransferFrom() |   32781  |  32643  |            31264           |
+|  safeTransferFrom() |   33146  |  33140  |            34139           |
+| supportsInterface() |   21983  |  21983  |            21960           |
+
+</details>
+
+<details><summary>ERC1155</summary>
+
+|         Function        | Rustmate | Solmate | OpenZeppelin Contracts 5.0 |
+|:-----------------------:|:--------:|:-------:|:--------------------------:|
+|       balanceOf()       |   28390  |  24631  |            24675           |
+|    isApprovedForAll()   |   28474  |  25022  |            25081           |
+|          uri()          |   24346  |  22291  |            24984           |
+|   setApprovalForAll()   |   51321  |  46581  |            46690           |
+|    safeTransferFrom()   |   30167  |  29793  |            31672           |
+| safeBatchTransferFrom() |   33192  |  32054  |            33363           |
+|     balanceOfBatch()    |   25094  |  22961  |            23735           |
+|   supportsInterface()   |   22006  |  22006  |            22058           |
+
+</details>
+
+<details><summary>ERC6909</summary>
+
+|       Function      | Rustmate | Solmate | OpenZeppelin Contracts 5.0 |
+|:-------------------:|:--------:|:-------:|:--------------------------:|
+|      transfer()     |   77615  |  28656  |             N/A            |
+|    transferFrom()   |   68799  |  29356  |             N/A            |
+|      approve()      |   52110  |  47430  |             N/A            |
+|    setOperator()    |   51152  |  46750  |             N/A            |
+| supportsInterface() |   22376  |  21962  |             N/A            |
 
 </details>
 

diff --git a/benchmark/erc1155_benchmark/.gas-snapshot b/benchmark/erc1155_benchmark/.gas-snapshot
@@ -1,8 +1,8 @@
-ERC1155:balanceOf(address,uint256) (gas: 28414)
-ERC1155:isApprovedForAll(address,address) (gas: 28510)
-ERC1155:uri(uint256) (gas: 24357)
-ERC1155:setApprovalForAll(address,bool) (gas: 51345)
+ERC1155:balanceOf(address,uint256) (gas: 24631)
+ERC1155:isApprovedForAll(address,address) (gas: 25022)
+ERC1155:uri(uint256) (gas: 22291)
+ERC1155:setApprovalForAll(address,bool) (gas: 46581)
 ERC1155:safeTransferFrom(address,address,uint256,uint256,bytes) (gas: )
 ERC1155:safeBatchTransferFrom(address,address,uint256[],uint256[],bytes) (gas: )
-ERC1155:balanceOfBatch(address[],uint256[]) (gas: 25111)
-ERC1155:supportsInterface(bytes4) (gas: )
+ERC1155:balanceOfBatch(address[],uint256[]) (gas: 22961)
+ERC1155:supportsInterface(bytes4) (gas: 22006)
diff --git a/benchmark/erc1155_benchmark/snapshot.py b/benchmark/erc1155_benchmark/snapshot.py
@@ -1,25 +1,43 @@
 import subprocess
+import argparse
 
-# Deploy contract and get address
-contract = subprocess.run(['cargo', 'stylus', 'deploy', '-e', 'http://localhost:8547', '--private-key', '0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80'], stdout=subprocess.PIPE).stdout.decode("utf-8")
-location = contract.find('Base')
-address = contract[location-49:location-7]
+# Initialize parser
+parser = argparse.ArgumentParser(
+                    prog='ERC1155 Benchmark',
+                    description='Run gas benchmark on ERC1155 implementations')
+
+# Adding optional argument
+parser.add_argument("-addr", "--address", help = "Use a specific contract address")
 
-functions = {
-    'balanceOf(address,uint256)': ['(uint256)', address, '1'],
-    'isApprovedForAll(address,address)': ['(bool)', address, address],
-    'uri(uint256)': ['(string)', '1'],
-    'setApprovalForAll(address,bool)': ['(bool)', address, 'true'],
-    # 'safeTransferFrom(address,address,uint256,uint256,bytes)': ['(bool)', address, address, '1', '1', '0x'],
-    # 'safeBatchTransferFrom(address,address,uint256[],uint256[],bytes)': ['(bool)', address, address, '[]', '[]', '0x'],
-    # 'balanceOfBatch(address[],uint256[])': ['(uint256[])', '[]', '[]'],
-    # 'supportsInterface(bytes4)': ['(bool)', '0x80ac58cd']
-}
+# Read arguments from command line
+args = parser.parse_args()
 
-# Run estimates and write to snapshot file
-with open('.gas-snapshot', 'w') as snapshot:
-    for function, args in functions.items():
-        gas = subprocess.run(['cast', 'estimate', address, function + args[0]] + args[1:] + ['--rpc-url', 'http://localhost:8547'], stdout=subprocess.PIPE).stdout.decode("utf-8")[:-1]
-        snapshot.write(f'ERC1155:{function} (gas: {gas})\n')
+def run_benchmark(address):
+    functions = {
+        'balanceOf(address,uint256)': ['(uint256)', address, '1'],
+        'isApprovedForAll(address,address)': ['(bool)', address, address],
+        'uri(uint256)': ['(string)', '1'],
+        'setApprovalForAll(address,bool)': ['(bool)', address, 'true'],
+        'safeTransferFrom(address,address,uint256,uint256,bytes)': ['(bool)', address, address, '1', '1', '0x'],
+        'safeBatchTransferFrom(address,address,uint256[],uint256[],bytes)': ['(bool)', address, address, '[]', '[]', '0x'],
+        'balanceOfBatch(address[],uint256[])': ['(uint256[])', '[]', '[]'],
+        'supportsInterface(bytes4)': ['(bool)', '0x80ac58cd']
+    }
 
-print(subprocess.run(['cat', '.gas-snapshot'], stdout=subprocess.PIPE).stdout.decode("utf-8"))
+    # Run estimates and write to snapshot file
+    with open('.gas-snapshot', 'w') as snapshot:
+        for function, args in functions.items():
+            gas = subprocess.run(['cast', 'estimate', address, function + args[0]] + args[1:] + ['--rpc-url', 'http://localhost:8547'], stdout=subprocess.PIPE).stdout.decode("utf-8")[:-1]
+            snapshot.write(f'ERC1155:{function} (gas: {gas})\n')
+
+    print(subprocess.run(['cat', '.gas-snapshot'], stdout=subprocess.PIPE).stdout.decode("utf-8"))
+
+if args.address:
+    print(f"Using {args.address}...")
+    address = args.address
+else:
+    contract = subprocess.run(['cargo', 'stylus', 'deploy', '-e', 'http://localhost:8547', '--private-key', '0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80'], stdout=subprocess.PIPE).stdout.decode("utf-8")
+    location = contract.find('Base')
+    address = contract[location-49:location-7]
+
+run_benchmark(address)
diff --git a/benchmark/erc20_benchmark/.gas-snapshot b/benchmark/erc20_benchmark/.gas-snapshot
@@ -1,10 +1,10 @@
-ERC20:name() (gas: 22614)
-ERC20:symbol() (gas: 22573)
-ERC20:decimals() (gas: 22497)
-ERC20:totalSupply() (gas: 25005)
-ERC20:balanceOf(address) (gas: 26015)
-ERC20:allowance(address,address) (gas: 27046)
-ERC20:nonces(address) (gas: 26004)
-ERC20:approve(address,uint256) (gas: 48940)
-ERC20:transfer(address,uint256) (gas: 71869)
-ERC20:transferFrom(address,address,uint256) (gas: 56707)
+ERC20:name() (gas: 23043)
+ERC20:symbol() (gas: 22974)
+ERC20:decimals() (gas: 22726)
+ERC20:totalSupply() (gas: 25617)
+ERC20:balanceOf(address) (gas: 26851)
+ERC20:allowance(address,address) (gas: 28263)
+ERC20:nonces(address) (gas: 26835)
+ERC20:approve(address,uint256) (gas: 50557)
+ERC20:transfer(address,uint256) (gas: 74234)
+ERC20:transferFrom(address,address,uint256) (gas: 60116)
diff --git a/benchmark/erc20_benchmark/snapshot.py b/benchmark/erc20_benchmark/snapshot.py
@@ -1,29 +1,47 @@
 import subprocess
+import argparse
 
-# Deploy contract and get address
-contract = subprocess.run(['cargo', 'stylus', 'deploy', '-e', 'http://localhost:8547', '--private-key', '0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80'], stdout=subprocess.PIPE).stdout.decode("utf-8")
-location = contract.find('Base')
-address = contract[location-49:location-7]
+# Initialize parser
+parser = argparse.ArgumentParser(
+                    prog='ERC20 Benchmark',
+                    description='Run gas benchmark on ERC20 implementations')
+
+# Adding optional argument
+parser.add_argument("-addr", "--address", help = "Use a specific contract address")
 
-functions = {
-    'name()': ['(string)'],
-    'symbol()': ['(string)'],
-    'decimals()': ['(uint8)'],
-    'totalSupply()': ['(uint256)'],
-    'balanceOf(address)': ['(uint256)', address],
-    'allowance(address,address)': ['(uint256)', address, address],
-    'nonces(address)': ['(uint256)', address],
-    'approve(address,uint256)': ['(bool)', address, '100'],
-    'transfer(address,uint256)': ['(bool)', address, '100'],
-    'transferFrom(address,address,uint256)': ['(bool)', address, address, '100'],
-    # 'permit(address,address,uint256,uint256,uint8,uint256,uint256)': ['(bool)', address, address, '100', '999', '0', '0', '0'],
-    # 'domainSeparator()': ['(bytes32)'],
-}
+# Read arguments from command line
+args = parser.parse_args()
 
-# Run estimates and write to snapshot file
-with open('.gas-snapshot', 'w') as snapshot:
-    for function, args in functions.items():
-        gas = subprocess.run(['cast', 'estimate', address, function + args[0]] + args[1:] + ['--rpc-url', 'http://localhost:8547'], stdout=subprocess.PIPE).stdout.decode("utf-8")[:-1]
-        snapshot.write(f'ERC20:{function} (gas: {gas})\n')
+def run_benchmark(address):
+    functions = {
+        'name()': ['(string)'],
+        'symbol()': ['(string)'],
+        'decimals()': ['(uint8)'],
+        'totalSupply()': ['(uint256)'],
+        'balanceOf(address)': ['(uint256)', address],
+        'allowance(address,address)': ['(uint256)', address, address],
+        'nonces(address)': ['(uint256)', address],
+        'approve(address,uint256)': ['(bool)', address, '100'],
+        'transfer(address,uint256)': ['(bool)', address, '100'],
+        'transferFrom(address,address,uint256)': ['(bool)', address, address, '100'],
+        # 'permit(address,address,uint256,uint256,uint8,uint256,uint256)': ['(bool)', address, address, '100', '999', '0', '0', '0'],
+        # 'domainSeparator()': ['(bytes32)'],
+    }
 
-print(subprocess.run(['cat', '.gas-snapshot'], stdout=subprocess.PIPE).stdout.decode("utf-8"))
+    # Run estimates and write to snapshot file
+    with open('.gas-snapshot', 'w') as snapshot:
+        for function, args in functions.items():
+            gas = subprocess.run(['cast', 'estimate', address, function + args[0]] + args[1:] + ['--rpc-url', 'http://localhost:8547'], stdout=subprocess.PIPE).stdout.decode("utf-8")[:-1]
+            snapshot.write(f'ERC20:{function} (gas: {gas})\n')
+
+    print(subprocess.run(['cat', '.gas-snapshot'], stdout=subprocess.PIPE).stdout.decode("utf-8"))
+
+if args.address:
+    print(f"Using {args.address}...")
+    address = args.address
+else:
+    contract = subprocess.run(['cargo', 'stylus', 'deploy', '-e', 'http://localhost:8547', '--private-key', '0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80'], stdout=subprocess.PIPE).stdout.decode("utf-8")
+    location = contract.find('Base')
+    address = contract[location-49:location-7]
+
+run_benchmark(address)
diff --git a/benchmark/erc20_benchmark/src/lib.rs b/benchmark/erc20_benchmark/src/lib.rs
@@ -24,7 +24,6 @@ sol_storage! {
     pub struct MyToken {
         #[borrow] // Allows erc20 to access MyToken's storage and make calls
         ERC20<SampleParams> erc20;
-        uint256 total_supply;
     }
 }
 

diff --git a/benchmark/erc6909_benchmark/.gas-snapshot b/benchmark/erc6909_benchmark/.gas-snapshot
@@ -1,5 +1,5 @@
-ERC6909:transfer(address,uint256,uint256) (gas: 77629)
-ERC6909:transferFrom(address,address,uint256,uint256) (gas: 68815)
-ERC6909:approve(address,uint256,uint256) (gas: 52124)
-ERC6909:setOperator(address,bool) (gas: 51164)
-ERC6909:supportsInterface(bytes4) (gas: )
+ERC6909:transfer(address,uint256,uint256) (gas: )
+ERC6909:transferFrom(address,address,uint256,uint256) (gas: )
+ERC6909:approve(address,uint256,uint256) (gas: 47430)
+ERC6909:setOperator(address,bool) (gas: 46750)
+ERC6909:supportsInterface(bytes4) (gas: 21962)
diff --git a/benchmark/erc6909_benchmark/snapshot.py b/benchmark/erc6909_benchmark/snapshot.py
@@ -1,22 +1,40 @@
 import subprocess
+import argparse
 
-# Deploy contract and get address
-contract = subprocess.run(['cargo', 'stylus', 'deploy', '-e', 'http://localhost:8547', '--private-key', '0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80'], stdout=subprocess.PIPE).stdout.decode("utf-8")
-location = contract.find('Base')
-address = contract[location-49:location-7]
+# Initialize parser
+parser = argparse.ArgumentParser(
+                    prog='ERC6909 Benchmark',
+                    description='Run gas benchmark on ERC6909 implementations')
+
+# Adding optional argument
+parser.add_argument("-addr", "--address", help = "Use a specific contract address")
 
-functions = {
-    'transfer(address,uint256,uint256)': ['(bool)', address, '1', '1'],
-    'transferFrom(address,address,uint256,uint256)': ['(bool)', address, address, '1', '1'],
-    'approve(address,uint256,uint256)': ['(bool)', address, '1', '1'],
-    'setOperator(address,bool)': ['(bool)', address, 'true'],
-    # 'supportsInterface(bytes4)': ['(bool)', '0x80ac58cd']
-}
+# Read arguments from command line
+args = parser.parse_args()
 
-# Run estimates and write to snapshot file
-with open('.gas-snapshot', 'w') as snapshot:
-    for function, args in functions.items():
-        gas = subprocess.run(['cast', 'estimate', address, function + args[0]] + args[1:] + ['--rpc-url', 'http://localhost:8547'], stdout=subprocess.PIPE).stdout.decode("utf-8")[:-1]
-        snapshot.write(f'ERC6909:{function} (gas: {gas})\n')
+def run_benchmark(address):
+    functions = {
+        'transfer(address,uint256,uint256)': ['(bool)', address, '1', '1'],
+        'transferFrom(address,address,uint256,uint256)': ['(bool)', address, address, '1', '1'],
+        'approve(address,uint256,uint256)': ['(bool)', address, '1', '1'],
+        'setOperator(address,bool)': ['(bool)', address, 'true'],
+        'supportsInterface(bytes4)': ['(bool)', '0x80ac58cd']
+    }
 
-print(subprocess.run(['cat', '.gas-snapshot'], stdout=subprocess.PIPE).stdout.decode("utf-8"))
+    # Run estimates and write to snapshot file
+    with open('.gas-snapshot', 'w') as snapshot:
+        for function, args in functions.items():
+            gas = subprocess.run(['cast', 'estimate', address, function + args[0]] + args[1:] + ['--rpc-url', 'http://localhost:8547'], stdout=subprocess.PIPE).stdout.decode("utf-8")[:-1]
+            snapshot.write(f'ERC6909:{function} (gas: {gas})\n')
+
+    print(subprocess.run(['cat', '.gas-snapshot'], stdout=subprocess.PIPE).stdout.decode("utf-8"))
+
+if args.address:
+    print(f"Using {args.address}...")
+    address = args.address
+else:
+    contract = subprocess.run(['cargo', 'stylus', 'deploy', '-e', 'http://localhost:8547', '--private-key', '0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80'], stdout=subprocess.PIPE).stdout.decode("utf-8")
+    location = contract.find('Base')
+    address = contract[location-49:location-7]
+
+run_benchmark(address)
diff --git a/benchmark/erc721_benchmark/.gas-snapshot b/benchmark/erc721_benchmark/.gas-snapshot
@@ -1,13 +1,13 @@
-ERC721:name() (gas: 23295)
-ERC721:symbol() (gas: 23234)
+ERC721:name() (gas: 23286)
+ERC721:symbol() (gas: 23225)
 ERC721:ownerOf(uint256) (gas: )
-ERC721:balanceOf(address) (gas: 27093)
-ERC721:getApproved(uint256) (gas: 26760)
-ERC721:isApprovedForAll(address,address) (gas: 28435)
-ERC721:tokenURI(uint256) (gas: 24304)
-ERC721:approve(address,uint256) (gas: )
+ERC721:balanceOf(address) (gas: 27094)
+ERC721:getApproved(uint256) (gas: 26749)
+ERC721:isApprovedForAll(address,address) (gas: 28447)
+ERC721:tokenURI(uint256) (gas: 24293)
+ERC721:approve(address,uint256) (gas: 48693)
 ERC721:setApprovalForAll(address,bool) (gas: 51279)
 ERC721:transferFrom(address,address,uint256) (gas: )
 ERC721:safeTransferFrom(address,address,uint256) (gas: )
 ERC721:safeTransferFrom(address,address,uint256,bytes) (gas: )
-ERC721:supportsInterface(bytes4) (gas: )
+ERC721:supportsInterface(bytes4) (gas: 21983)