Merge pull request #738 from clrfund/feat/tally-subtask

Refactor tally hardhat script

.github/workflows/finalize-round.yml

@@ -29,6 +29,8 @@ env:
   CIRCUIT_TYPE: micro
   ZKEYS_DOWNLOAD_SCRIPT: "download-6-9-2-3.sh"
   JSONRPC_HTTP_URL: ${{ github.event.inputs.jsonrpc_url }}
+  PINATA_API_KEY: ${{ secrets.PINATA_API_KEY }}
+  PINATA_SECRET_API_KEY: ${{ secrets.PINATA_SECRET_API_KEY }}
 
 jobs:
   finalize:
@@ -84,10 +86,9 @@ jobs:
         mkdir -p proof_output
         yarn hardhat tally --clrfund "${CLRFUND_ADDRESS}" --network "${NETWORK}" \
           --rapidsnark ${RAPID_SNARK} \
-          --circuit-directory ${CIRCUIT_DIRECTORY} \
+          --params-dir ${CIRCUIT_DIRECTORY} \
           --blocks-per-batch ${BLOCKS_PER_BATCH} \
-          --maci-tx-hash "${MACI_TX_HASH}" --output-dir "./proof_output"
-        curl --location --request POST 'https://api.pinata.cloud/pinning/pinFileToIPFS' \
-          --header "Authorization: Bearer ${{ secrets.PINATA_JWT }}" \
-          --form 'file=@"./proof_output/tally.json"'
-        yarn hardhat --network "${NETWORK}" finalize --clrfund "${CLRFUND_ADDRESS}"
+          --maci-tx-hash "${MACI_TX_HASH}" \
+          --proof-dir "./proof_output"
+        yarn hardhat --network "${NETWORK}" finalize --clrfund "${CLRFUND_ADDRESS}" \
+          --proof-dir "./proof_output"

.github/workflows/test-scripts.yml

@@ -10,6 +10,8 @@ on:
 env:
   NODE_VERSION: 20.x
   ZKEYS_DOWNLOAD_SCRIPT: "download-6-9-2-3.sh"
+  PINATA_API_KEY: ${{ secrets.PINATA_API_KEY }}
+  PINATA_SECRET_API_KEY: ${{ secrets.PINATA_SECRET_API_KEY }}
 
 jobs:
   script-tests:

contracts/.env.example

@@ -6,13 +6,17 @@ JSONRPC_HTTP_URL=https://eth-goerli.alchemyapi.io/v2/ADD_API_KEY
 WALLET_MNEMONIC=
 WALLET_PRIVATE_KEY=
 
-# The coordinator MACI private key, required by the tally script
+# The coordinator MACI private key, required by the gen-proofs script
 COORDINATOR_MACISK=
 
 # API key used to verify contracts on arbitrum chain (including testnet)
 # Update the etherscan section in hardhat.config to add API key for other chains
 ARBISCAN_API_KEY=
 
+# PINATE credentials to upload tally.json file to IPFS; used by the tally script
+PINATA_API_KEY=
+PINATA_SECRET_API_KEY=
+
 # these are used in the e2e testing
 CIRCUIT_TYPE=
 CIRCUIT_DIRECTORY=

contracts/e2e/index.ts

@@ -36,6 +36,7 @@ import path from 'path'
 import { FundingRound } from '../typechain-types'
 import { JSONFile } from '../utils/JSONFile'
 import { EContracts } from '../utils/types'
+import { getTalyFilePath } from '../utils/misc'
 
 type VoteData = { recipientIndex: number; voiceCredits: bigint }
 type ClaimData = { [index: number]: bigint }
@@ -359,6 +360,8 @@ describe('End-to-end Tests', function () {
       mkdirSync(outputDir, { recursive: true })
     }
 
+    const tallyFile = getTalyFilePath(outputDir)
+
     // past an end block that's later than the MACI start block
     const genProofArgs = getGenProofArgs({
       maciAddress,
@@ -368,6 +371,7 @@ describe('End-to-end Tests', function () {
       circuitType: circuit,
       circuitDirectory,
       outputDir,
+      tallyFile,
       blocksPerBatch: DEFAULT_GET_LOG_BATCH_SIZE,
       maciTxHash: maciTransactionHash,
       signer: coordinator,

contracts/package.json

@@ -16,6 +16,7 @@
   },
   "dependencies": {
     "@openzeppelin/contracts": "4.9.0",
+    "@pinata/sdk": "^2.1.0",
     "dotenv": "^8.2.0",
     "maci-contracts": "^1.2.0",
     "solidity-rlp": "2.0.8"

contracts/sh/runScriptTests.sh

@@ -33,17 +33,17 @@ yarn hardhat contribute --network ${HARDHAT_NETWORK}
 
 yarn hardhat time-travel --seconds ${ROUND_DURATION} --network ${HARDHAT_NETWORK}
 
-# run the tally script
+# tally the votes
 NODE_OPTIONS="--max-old-space-size=4096"
 yarn hardhat tally \
   --rapidsnark ${RAPID_SNARK} \
-  --batch-size 8 \
-  --output-dir ${OUTPUT_DIR} \
+  --proof-dir ${OUTPUT_DIR} \
+  --maci-start-block 0 \
   --network "${HARDHAT_NETWORK}"
 
 # finalize the round
-yarn hardhat finalize --tally-file ${TALLY_FILE} --network ${HARDHAT_NETWORK}
+yarn hardhat finalize --proof-dir ${OUTPUT_DIR} --network ${HARDHAT_NETWORK}
 
 # claim funds
-yarn hardhat claim --recipient 1 --tally-file ${TALLY_FILE}  --network ${HARDHAT_NETWORK}
-yarn hardhat claim --recipient 2 --tally-file ${TALLY_FILE}  --network ${HARDHAT_NETWORK}
+yarn hardhat claim --recipient 1 --proof-dir ${OUTPUT_DIR}  --network ${HARDHAT_NETWORK}
+yarn hardhat claim --recipient 2 --proof-dir ${OUTPUT_DIR}  --network ${HARDHAT_NETWORK}

contracts/tasks/index.ts

@@ -23,3 +23,7 @@ import './runners/addRecipients'
 import './runners/findStorageSlot'
 import './runners/verifyTallyFile'
 import './runners/verifyAll'
+import './runners/genProofs'
+import './runners/proveOnChain'
+import './runners/publishTallyResults'
+import './runners/resetTally'

contracts/tasks/runners/claim.ts

@@ -2,75 +2,87 @@
  * Claim funds. This script is mainly used by e2e testing
  *
  * Sample usage:
- *  yarn hardhat claim \
- *   --tally-file <tally file> \
- *   --recipient <recipient-index> \
- *   --network <network>
+ *  yarn hardhat claim --recipient <recipient-index> --network <network>
  */
 
 import { getEventArg } from '../../utils/contracts'
 import { getRecipientClaimData } from '@clrfund/common'
 import { JSONFile } from '../../utils/JSONFile'
-import { isPathExist } from '../../utils/misc'
+import {
+  getProofDirForRound,
+  getTalyFilePath,
+  isPathExist,
+} from '../../utils/misc'
 import { getNumber } from 'ethers'
 import { task, types } from 'hardhat/config'
 import { EContracts } from '../../utils/types'
 import { ContractStorage } from '../helpers/ContractStorage'
 
 task('claim', 'Claim funnds for test recipients')
+  .addOptionalParam('roundAddress', 'Funding round contract address')
   .addParam(
     'recipient',
     'The recipient index in the tally file',
     undefined,
     types.int
   )
-  .addParam('tallyFile', 'The tally file')
-  .setAction(async ({ tallyFile, recipient }, { ethers, network }) => {
-    if (!isPathExist(tallyFile)) {
-      throw new Error(`Path ${tallyFile} does not exist`)
-    }
+  .addParam('proofDir', 'The proof output directory', './proof_output')
+  .setAction(
+    async ({ proofDir, recipient, roundAddress }, { ethers, network }) => {
+      if (recipient <= 0) {
+        throw new Error('Recipient must be greater than 0')
+      }
 
-    if (recipient <= 0) {
-      throw new Error('Recipient must be greater than 0')
-    }
+      const storage = ContractStorage.getInstance()
+      const fundingRound =
+        roundAddress ??
+        storage.mustGetAddress(EContracts.FundingRound, network.name)
 
-    const storage = ContractStorage.getInstance()
-    const fundingRound = storage.mustGetAddress(
-      EContracts.FundingRound,
-      network.name
-    )
+      const proofDirForRound = getProofDirForRound(
+        proofDir,
+        network.name,
+        fundingRound
+      )
 
-    const tally = JSONFile.read(tallyFile)
+      const tallyFile = getTalyFilePath(proofDirForRound)
+      if (!isPathExist(tallyFile)) {
+        throw new Error(`Path ${tallyFile} does not exist`)
+      }
 
-    const fundingRoundContract = await ethers.getContractAt(
-      EContracts.FundingRound,
-      fundingRound
-    )
+      const tally = JSONFile.read(tallyFile)
 
-    const recipientStatus = await fundingRoundContract.recipients(recipient)
-    if (recipientStatus.fundsClaimed) {
-      throw new Error(`Recipient already claimed funds`)
-    }
+      const fundingRoundContract = await ethers.getContractAt(
+        EContracts.FundingRound,
+        fundingRound
+      )
 
-    const pollAddress = await fundingRoundContract.poll()
-    console.log('pollAddress', pollAddress)
+      const recipientStatus = await fundingRoundContract.recipients(recipient)
+      if (recipientStatus.fundsClaimed) {
+        throw new Error(`Recipient already claimed funds`)
+      }
 
-    const poll = await ethers.getContractAt(EContracts.Poll, pollAddress)
-    const treeDepths = await poll.treeDepths()
-    const recipientTreeDepth = getNumber(treeDepths.voteOptionTreeDepth)
+      const pollAddress = await fundingRoundContract.poll()
+      console.log('pollAddress', pollAddress)
 
-    // Claim funds
-    const recipientClaimData = getRecipientClaimData(
-      recipient,
-      recipientTreeDepth,
-      tally
-    )
-    const claimTx = await fundingRoundContract.claimFunds(...recipientClaimData)
-    const claimedAmount = await getEventArg(
-      claimTx,
-      fundingRoundContract,
-      'FundsClaimed',
-      '_amount'
-    )
-    console.log(`Recipient ${recipient} claimed ${claimedAmount} tokens.`)
-  })
+      const poll = await ethers.getContractAt(EContracts.Poll, pollAddress)
+      const treeDepths = await poll.treeDepths()
+      const recipientTreeDepth = getNumber(treeDepths.voteOptionTreeDepth)
+
+      // Claim funds
+      const recipientClaimData = getRecipientClaimData(
+        recipient,
+        recipientTreeDepth,
+        tally
+      )
+      const claimTx = await fundingRoundContract.claimFunds(
+        ...recipientClaimData
+      )
+      const claimedAmount = await getEventArg(
+        claimTx,
+        fundingRoundContract,
+        'FundsClaimed',
+        '_amount'
+      )
+      console.log(`Recipient ${recipient} claimed ${claimedAmount} tokens.`)
+    }
+  )

contracts/tasks/runners/exportRound.ts

@@ -29,6 +29,11 @@ type RoundListEntry = {
   isFinalized: boolean
 }
 
+// Map hardhat network name to the etherscan api network name in the hardhat.config
+const ETHERSCAN_NETWORKS: Record<string, string> = {
+  optimism: 'optimisticEthereum',
+}
+
 const toUndefined = () => undefined
 const toString = (val: bigint) => BigInt(val).toString()
 const toZero = () => BigInt(0)
@@ -41,13 +46,18 @@ function roundListFileName(directory: string): string {
   return path.join(directory, 'rounds.json')
 }
 
+function toEtherscanNetworkName(network: string): string {
+  return ETHERSCAN_NETWORKS[network] ?? network
+}
+
 function getEtherscanApiKey(config: any, network: string): string {
   let etherscanApiKey = ''
   if (config.etherscan?.apiKey) {
     if (typeof config.etherscan.apiKey === 'string') {
       etherscanApiKey = config.etherscan.apiKey
     } else {
-      etherscanApiKey = config.etherscan.apiKey[network]
+      const etherscanNetwork = toEtherscanNetworkName(network)
+      etherscanApiKey = config.etherscan.apiKey[etherscanNetwork]
     }
   }
 

contracts/tasks/runners/finalize.ts

@@ -6,7 +6,7 @@
  *   - clrfund owner's wallet private key to interact with the contract
  *
  * Sample usage:
- *  yarn hardhat finalize --clrfund <clrfund address> --tally-file <tally file> --network <network>
+ *  yarn hardhat finalize --clrfund <clrfund address> --network <network>
  */
 
 import { JSONFile } from '../../utils/JSONFile'
@@ -16,20 +16,13 @@ import { task } from 'hardhat/config'
 import { EContracts } from '../../utils/types'
 import { ContractStorage } from '../helpers/ContractStorage'
 import { Subtask } from '../helpers/Subtask'
+import { getProofDirForRound, getTalyFilePath } from '../../utils/misc'
 
 task('finalize', 'Finalize a funding round')
   .addOptionalParam('clrfund', 'The ClrFund contract address')
-  .addOptionalParam(
-    'tallyFile',
-    'The tally file path',
-    './proof_output/tally.json'
-  )
-  .setAction(async ({ clrfund, tallyFile }, hre) => {
+  .addParam('proofDir', 'The proof output directory', './proof_output')
+  .setAction(async ({ clrfund, proofDir }, hre) => {
     const { ethers, network } = hre
-    const tally = JSONFile.read(tallyFile)
-    if (!tally.maci) {
-      throw Error('Bad tally file ' + tallyFile)
-    }
 
     const storage = ContractStorage.getInstance()
     const subtask = Subtask.getInstance(hre)
@@ -63,6 +56,17 @@ task('finalize', 'Finalize a funding round')
     const treeDepths = await pollContract.treeDepths()
     console.log('voteOptionTreeDepth', treeDepths.voteOptionTreeDepth)
 
+    const currentRoundProofDir = getProofDirForRound(
+      proofDir,
+      network.name,
+      currentRoundAddress
+    )
+    const tallyFile = getTalyFilePath(currentRoundProofDir)
+    const tally = JSONFile.read(tallyFile)
+    if (!tally.maci) {
+      throw Error('Bad tally file ' + tallyFile)
+    }
+
     const totalSpent = tally.totalSpentVoiceCredits.spent
     const totalSpentSalt = tally.totalSpentVoiceCredits.salt