refactor the tally script to re-run without extra inputs

.github/workflows/finalize-round.yml

@@ -86,8 +86,7 @@ jobs:
           --rapidsnark ${RAPID_SNARK} \
           --params-dir ${CIRCUIT_DIRECTORY} \
           --blocks-per-batch ${BLOCKS_PER_BATCH} \
-          --maci-tx-hash "${MACI_TX_HASH}" --output-dir "./proof_output"
-        curl --location --request POST 'https://api.pinata.cloud/pinning/pinFileToIPFS' \
-          --header "Authorization: Bearer ${{ secrets.PINATA_JWT }}" \
-          --form 'file=@"./proof_output/tally.json"'
-        yarn hardhat --network "${NETWORK}" finalize --clrfund "${CLRFUND_ADDRESS}"
+          --maci-tx-hash "${MACI_TX_HASH}" \
+          --proof-dir "./proof_output"
+        yarn hardhat --network "${NETWORK}" finalize --clrfund "${CLRFUND_ADDRESS}" \
+          --proof-dir "./proof_output"

contracts/.env.example

@@ -13,6 +13,10 @@ COORDINATOR_MACISK=
 # Update the etherscan section in hardhat.config to add API key for other chains
 ARBISCAN_API_KEY=
 
+# PINATE credentials to upload tally.json file to IPFS; used by the tally script
+PINATA_API_KEY=
+PINATA_SECRET_API_KEY=
+
 # these are used in the e2e testing
 CIRCUIT_TYPE=
 CIRCUIT_DIRECTORY=

contracts/e2e/index.ts

@@ -36,6 +36,7 @@ import path from 'path'
 import { FundingRound } from '../typechain-types'
 import { JSONFile } from '../utils/JSONFile'
 import { EContracts } from '../utils/types'
+import { getTalyFilePath } from '../utils/misc'
 
 type VoteData = { recipientIndex: number; voiceCredits: bigint }
 type ClaimData = { [index: number]: bigint }
@@ -359,6 +360,8 @@ describe('End-to-end Tests', function () {
       mkdirSync(outputDir, { recursive: true })
     }
 
+    const tallyFile = getTalyFilePath(outputDir)
+
     // past an end block that's later than the MACI start block
     const genProofArgs = getGenProofArgs({
       maciAddress,
@@ -368,6 +371,7 @@ describe('End-to-end Tests', function () {
       circuitType: circuit,
       circuitDirectory,
       outputDir,
+      tallyFile,
       blocksPerBatch: DEFAULT_GET_LOG_BATCH_SIZE,
       maciTxHash: maciTransactionHash,
       signer: coordinator,

contracts/package.json

@@ -16,6 +16,7 @@
   },
   "dependencies": {
     "@openzeppelin/contracts": "4.9.0",
+    "@pinata/sdk": "^2.1.0",
     "dotenv": "^8.2.0",
     "maci-contracts": "^1.2.0",
     "solidity-rlp": "2.0.8"

contracts/sh/runScriptTests.sh

@@ -35,17 +35,15 @@ yarn hardhat time-travel --seconds ${ROUND_DURATION} --network ${HARDHAT_NETWORK
 
 # tally the votes
 NODE_OPTIONS="--max-old-space-size=4096"
-yarn hardhat gen-proofs \
+yarn hardhat tally \
   --rapidsnark ${RAPID_SNARK} \
   --proof-dir ${OUTPUT_DIR} \
   --maci-start-block 0 \
   --network "${HARDHAT_NETWORK}"
-yarn hardhat prove-on-chain --proof-dir ${OUTPUT_DIR} --network "${HARDHAT_NETWORK}"
-yarn hardhat publish-tally-results --proof-dir ${OUTPUT_DIR} --network "${HARDHAT_NETWORK}"
 
 # finalize the round
-yarn hardhat finalize --tally-file ${TALLY_FILE} --network ${HARDHAT_NETWORK}
+yarn hardhat finalize --proof-dir ${OUTPUT_DIR} --network ${HARDHAT_NETWORK}
 
 # claim funds
-yarn hardhat claim --recipient 1 --tally-file ${TALLY_FILE}  --network ${HARDHAT_NETWORK}
-yarn hardhat claim --recipient 2 --tally-file ${TALLY_FILE}  --network ${HARDHAT_NETWORK}
+yarn hardhat claim --recipient 1 --proof-dir ${OUTPUT_DIR}  --network ${HARDHAT_NETWORK}
+yarn hardhat claim --recipient 2 --proof-dir ${OUTPUT_DIR}  --network ${HARDHAT_NETWORK}

contracts/tasks/runners/claim.ts

@@ -2,16 +2,17 @@
  * Claim funds. This script is mainly used by e2e testing
  *
  * Sample usage:
- *  yarn hardhat claim \
- *   --tally-file <tally file> \
- *   --recipient <recipient-index> \
- *   --network <network>
+ *  yarn hardhat claim --recipient <recipient-index> --network <network>
  */
 
 import { getEventArg } from '../../utils/contracts'
 import { getRecipientClaimData } from '@clrfund/common'
 import { JSONFile } from '../../utils/JSONFile'
-import { isPathExist } from '../../utils/misc'
+import {
+  getProofDirForRound,
+  getTalyFilePath,
+  isPathExist,
+} from '../../utils/misc'
 import { getNumber } from 'ethers'
 import { task, types } from 'hardhat/config'
 import { EContracts } from '../../utils/types'
@@ -25,13 +26,9 @@ task('claim', 'Claim funnds for test recipients')
     undefined,
     types.int
   )
-  .addParam('tallyFile', 'The tally file')
+  .addParam('proofDir', 'The proof output directory', './proof_output')
   .setAction(
-    async ({ tallyFile, recipient, roundAddress }, { ethers, network }) => {
-      if (!isPathExist(tallyFile)) {
-        throw new Error(`Path ${tallyFile} does not exist`)
-      }
-
+    async ({ proofDir, recipient, roundAddress }, { ethers, network }) => {
       if (recipient <= 0) {
         throw new Error('Recipient must be greater than 0')
       }
@@ -41,6 +38,17 @@ task('claim', 'Claim funnds for test recipients')
         roundAddress ??
         storage.mustGetAddress(EContracts.FundingRound, network.name)
 
+      const proofDirForRound = getProofDirForRound(
+        proofDir,
+        network.name,
+        fundingRound
+      )
+
+      const tallyFile = getTalyFilePath(proofDirForRound)
+      if (!isPathExist(tallyFile)) {
+        throw new Error(`Path ${tallyFile} does not exist`)
+      }
+
       const tally = JSONFile.read(tallyFile)
 
       const fundingRoundContract = await ethers.getContractAt(

contracts/tasks/runners/finalize.ts

@@ -6,7 +6,7 @@
  *   - clrfund owner's wallet private key to interact with the contract
  *
  * Sample usage:
- *  yarn hardhat finalize --clrfund <clrfund address> --tally-file <tally file> --network <network>
+ *  yarn hardhat finalize --clrfund <clrfund address> --network <network>
  */
 
 import { JSONFile } from '../../utils/JSONFile'
@@ -16,20 +16,13 @@ import { task } from 'hardhat/config'
 import { EContracts } from '../../utils/types'
 import { ContractStorage } from '../helpers/ContractStorage'
 import { Subtask } from '../helpers/Subtask'
+import { getProofDirForRound, getTalyFilePath } from '../../utils/misc'
 
 task('finalize', 'Finalize a funding round')
   .addOptionalParam('clrfund', 'The ClrFund contract address')
-  .addOptionalParam(
-    'tallyFile',
-    'The tally file path',
-    './proof_output/tally.json'
-  )
-  .setAction(async ({ clrfund, tallyFile }, hre) => {
+  .addParam('proofDir', 'The proof output directory', './proof_output')
+  .setAction(async ({ clrfund, proofDir }, hre) => {
     const { ethers, network } = hre
-    const tally = JSONFile.read(tallyFile)
-    if (!tally.maci) {
-      throw Error('Bad tally file ' + tallyFile)
-    }
 
     const storage = ContractStorage.getInstance()
     const subtask = Subtask.getInstance(hre)
@@ -63,6 +56,17 @@ task('finalize', 'Finalize a funding round')
     const treeDepths = await pollContract.treeDepths()
     console.log('voteOptionTreeDepth', treeDepths.voteOptionTreeDepth)
 
+    const currentRoundProofDir = getProofDirForRound(
+      proofDir,
+      network.name,
+      currentRoundAddress
+    )
+    const tallyFile = getTalyFilePath(currentRoundProofDir)
+    const tally = JSONFile.read(tallyFile)
+    if (!tally.maci) {
+      throw Error('Bad tally file ' + tallyFile)
+    }
+
     const totalSpent = tally.totalSpentVoiceCredits.spent
     const totalSpentSalt = tally.totalSpentVoiceCredits.salt
 

contracts/tasks/runners/genProofs.ts

@@ -36,6 +36,29 @@ import { Subtask } from '../helpers/Subtask'
 import { getCurrentFundingRoundContract } from '../../utils/contracts'
 import { ContractStorage } from '../helpers/ContractStorage'
 import { DEFAULT_CIRCUIT } from '../../utils/circuits'
+import { JSONFile } from '../../utils/JSONFile'
+
+/**
+ * Check if the tally file with the maci contract address exists
+ * @param tallyFile The tally file path
+ * @param maciAddress The MACI contract address
+ * @returns true if the file exists and it contains the MACI contract address
+ */
+function tallyFileExists(tallyFile: string, maciAddress: string): boolean {
+  if (!isPathExist(tallyFile)) {
+    return false
+  }
+  try {
+    const tallyData = JSONFile.read(tallyFile)
+    return (
+      tallyData.maci &&
+      tallyData.maci.toLowerCase() === maciAddress.toLowerCase()
+    )
+  } catch {
+    // in case the file does not have the expected format/field
+    return false
+  }
+}
 
 task('gen-proofs', 'Generate MACI proofs offchain')
   .addOptionalParam('clrfund', 'FundingRound contract address')
@@ -148,46 +171,49 @@ task('gen-proofs', 'Generate MACI proofs offchain')
       const maciStateFile = getMaciStateFilePath(proofDir)
       const providerUrl = (network.config as any).url
 
-      if (!isPathExist(tallyFile)) {
-        if (!isPathExist(maciStateFile)) {
-          if (!maciTxHash && maciStartBlock == null) {
-            throw new Error(
-              'Please provide a value for --maci-tx-hash or --maci-start-block'
-            )
-          }
-
-          await genLocalState({
-            quiet,
-            outputPath: maciStateFile,
-            pollId,
-            maciContractAddress: maciAddress,
-            coordinatorPrivateKey: coordinatorMacisk,
-            ethereumProvider: providerUrl,
-            transactionHash: maciTxHash,
-            startBlock: maciStartBlock,
-            blockPerBatch: blocksPerBatch,
-            signer: coordinator,
-            sleep,
-          })
+      if (tallyFileExists(tallyFile, maciAddress)) {
+        console.log('The tally file has already been generated.')
+        return
+      }
+
+      if (!isPathExist(maciStateFile)) {
+        if (!maciTxHash && maciStartBlock == null) {
+          throw new Error(
+            'Please provide a value for --maci-tx-hash or --maci-start-block'
+          )
         }
 
-        const genProofArgs = getGenProofArgs({
-          maciAddress,
+        await genLocalState({
+          quiet,
+          outputPath: maciStateFile,
           pollId,
-          coordinatorMacisk,
-          rapidsnark,
-          circuitType: circuit,
-          circuitDirectory,
-          outputDir: proofDir,
-          blocksPerBatch: getNumber(blocksPerBatch),
-          maciStateFile,
-          tallyFile,
+          maciContractAddress: maciAddress,
+          coordinatorPrivateKey: coordinatorMacisk,
+          ethereumProvider: providerUrl,
+          transactionHash: maciTxHash,
+          startBlock: maciStartBlock,
+          blockPerBatch: blocksPerBatch,
           signer: coordinator,
-          quiet,
+          sleep,
         })
-        await genProofs(genProofArgs)
       }
 
+      const genProofArgs = getGenProofArgs({
+        maciAddress,
+        pollId,
+        coordinatorMacisk,
+        rapidsnark,
+        circuitType: circuit,
+        circuitDirectory,
+        outputDir: proofDir,
+        blocksPerBatch: getNumber(blocksPerBatch),
+        maciStateFile,
+        tallyFile,
+        signer: coordinator,
+        quiet,
+      })
+      await genProofs(genProofArgs)
+
       const success = true
       await subtask.finish(success)
     }

contracts/tasks/runners/publishTallyResults.ts

@@ -5,6 +5,8 @@
  * Make sure to set the following environment variables in the .env file
  * 1) WALLET_PRIVATE_KEY or WALLET_MNEMONIC
  *   - coordinator's wallet private key to interact with contracts
+ * 2) PINATA_API_KEY - The Pinata api key for pinning file to IPFS
+ * 3) PINATA_SECRET_API_KEY - The Pinata secret api key for pinning file to IPFS
  *
  * Sample usage:
  *
@@ -15,7 +17,7 @@
 import { BaseContract, getNumber, NonceManager } from 'ethers'
 import { task, types } from 'hardhat/config'
 
-import { getIpfsHash } from '../../utils/ipfs'
+import { Ipfs } from '../../utils/ipfs'
 import { JSONFile } from '../../utils/JSONFile'
 import { addTallyResultsBatch, TallyData, verify } from '../../utils/maci'
 import { FundingRound, Poll } from '../../typechain-types'
@@ -25,17 +27,17 @@ import { Subtask } from '../helpers/Subtask'
 import { getCurrentFundingRoundContract } from '../../utils/contracts'
 import { getTalyFilePath } from '../../utils/misc'
 import { ContractStorage } from '../helpers/ContractStorage'
+import { PINATA_PINNING_URL } from '../../utils/constants'
 
 /**
  * Publish the tally IPFS hash on chain if it's not already published
  * @param fundingRoundContract Funding round contract
- * @param tallyData Tally data
+ * @param tallyHash Tally hash
  */
 async function publishTallyHash(
   fundingRoundContract: FundingRound,
-  tallyData: TallyData
+  tallyHash: string
 ) {
-  const tallyHash = await getIpfsHash(tallyData)
   console.log(`Tally hash is ${tallyHash}`)
 
   const tallyHashOnChain = await fundingRoundContract.tallyHash()
@@ -63,7 +65,9 @@ async function submitTallyResults(
 ) {
   const startIndex = await fundingRoundContract.totalTallyResults()
   const total = tallyData.results.tally.length
-  console.log('Uploading tally results in batches of', batchSize)
+  if (startIndex < total) {
+    console.log('Uploading tally results in batches of', batchSize)
+  }
   const addTallyGas = await addTallyResultsBatch(
     fundingRoundContract,
     recipientTreeDepth,
@@ -119,6 +123,16 @@ task('publish-tally-results', 'Publish tally results')
       const coordinator = manageNonce ? new NonceManager(signer) : signer
       console.log('Coordinator address: ', await coordinator.getAddress())
 
+      const apiKey = process.env.PINATA_API_KEY
+      if (!apiKey) {
+        throw new Error('Env. variable PINATA_API_KEY not set')
+      }
+
+      const secretApiKey = process.env.PINATA_SECRET_API_KEY
+      if (!secretApiKey) {
+        throw new Error('Env. variable PINATA_SECRET_API_KEY not set')
+      }
+
       await subtask.logStart()
 
       const clrfundContractAddress =
@@ -149,8 +163,10 @@ task('publish-tally-results', 'Publish tally results')
         quiet,
       })
 
+      const tallyHash = await Ipfs.pinFile(tallyFile, apiKey, secretApiKey)
+
       // Publish tally hash if it is not already published
-      await publishTallyHash(fundingRoundContract, tallyData)
+      await publishTallyHash(fundingRoundContract, tallyHash)
 
       // Submit tally results to the funding round contract
       // This function can be re-run from where it left off