From abd37866989144e6896149cbc0b9a34024482689 Mon Sep 17 00:00:00 2001 From: Sinka Date: Tue, 19 Nov 2024 00:04:54 +0800 Subject: [PATCH] add sketch of submit mode auto --- ts/src/config.ts | 4 + ts/src/lib.ts | 29 ++---- ts/src/reproduce.ts | 2 +- ts/src/service.ts | 22 +++- ts/src/settle.ts | 245 ++++++++++++++++++++++++++++---------------- 5 files changed, 191 insertions(+), 111 deletions(-) diff --git a/ts/src/config.ts b/ts/src/config.ts index 8bcc5c4..b11a32f 100644 --- a/ts/src/config.ts +++ b/ts/src/config.ts @@ -61,6 +61,10 @@ export const get_contract_addr = () => { } } +export const get_chain_id = () => { + return 16; +} + export const get_user_addr = () => { if (process.env.USER_ADDRESS) { return process.env.USER_ADDRESS; diff --git a/ts/src/lib.ts b/ts/src/lib.ts index bf85970..9a97c36 100644 --- a/ts/src/lib.ts +++ b/ts/src/lib.ts @@ -31,29 +31,18 @@ export class U8ArrayUtil { } } -export class NumberUtil { - bn: BN; - constructor(num: number) { - this.bn = new BN(num); - } - toBN(){ - let bns = new Array(); - let bnStr = this.bn.toString("hex", 64) - for (let i = 0; i < bnStr.length; i += 16) { - const chunk = bnStr.slice(i, i + 16); - let a = new BN(chunk, 'hex', 'be'); - bns.push(a); - } - return bns; - } - toU64StringArray() { - return this.toBN().map((x) => x.toString(10)); - } -} - export function merkleRootToBeHexString(root: BigUint64Array) { let bigint = root[3] + (root[2]<<64n) + (root[1]<<128n) + (root[0]<<192n); let bnStr = bigint.toString(10); let bn = new BN(bnStr, 10); return '0x' + bn.toString("hex", 64); } + +export function hexStringToMerkleRoot(hexStr: string) { + let merkleRootBN = BigInt(hexStr); + let root = [0n, 0n, 0n, 0n]; + for (let i=0; i<4; i++) { + root[i] = (merkleRootBN >> BigInt(64 * (3-i))) % (1n << 64n); + } + return root; +} diff --git a/ts/src/reproduce.ts b/ts/src/reproduce.ts index 3667ac2..7ae0561 100644 --- a/ts/src/reproduce.ts +++ b/ts/src/reproduce.ts @@ -2,7 +2,7 @@ import BN from "bn.js"; import { ServiceHelper, get_contract_addr, modelBundle, get_user_private_account } from "./config.js"; import abiData from './Proxy.json' assert { type: 'json' }; import {ZkWasmUtil, PaginationResult, QueryParams, Task, VerifyProofParams} from "zkwasm-service-helper"; -import { U8ArrayUtil, NumberUtil } from './lib.js'; +import { U8ArrayUtil } from './lib.js'; import * as fs from 'fs'; diff --git a/ts/src/service.ts b/ts/src/service.ts index eb5ba2d..c665a64 100644 --- a/ts/src/service.ts +++ b/ts/src/service.ts @@ -13,7 +13,7 @@ import { getMerkleArray } from "./settle.js"; import { ZkWasmUtil } from "zkwasm-service-helper"; import dotenv from 'dotenv'; import mongoose from 'mongoose'; -import {merkleRootToBeHexString} from "./lib.js"; +import {hexStringToMerkleRoot, merkleRootToBeHexString} from "./lib.js"; import {sha256} from "ethers"; // Load environment variables from .env file @@ -378,9 +378,7 @@ async function main() { }); app.post('/query', async (req, res) => { - console.log("receive query command"); const value = req.body; - console.log("value is", value); if (!value) { return res.status(400).send('Value is required'); } @@ -417,6 +415,24 @@ async function main() { } }); + app.get('/prooftask/:root', async (req, res) => { + try { + let merkleRootString = req.params.root; + let merkleRoot = new BigUint64Array(hexStringToMerkleRoot(merkleRootString)); + let record = await modelBundle.findOne({ merkleRoot: merkleRoot}); + if (record) { + return res.status(201).json(record); + } else { + throw Error("TaskNotFound"); + } + } catch (err) { + // job not tracked + console.log(err); + res.status(500).json({ message: (err as Error).toString()}); + } + }); + + app.post('/config', async (req, res) => { try { let jstr = application.get_config(); diff --git a/ts/src/settle.ts b/ts/src/settle.ts index a346b24..719432b 100644 --- a/ts/src/settle.ts +++ b/ts/src/settle.ts @@ -1,10 +1,10 @@ import BN from "bn.js"; import { ethers } from "ethers"; -import { ServiceHelper, get_mongoose_db, get_contract_addr, get_image_md5, modelBundle, get_settle_private_account } from "./config.js"; +import { ServiceHelper, get_mongoose_db, get_contract_addr, get_image_md5, modelBundle, get_settle_private_account, get_chain_id } from "./config.js"; import abiData from './Proxy.json' assert { type: 'json' }; import mongoose from 'mongoose'; -import {ZkWasmUtil, PaginationResult, QueryParams, Task, VerifyProofParams} from "zkwasm-service-helper"; -import { U8ArrayUtil, NumberUtil } from './lib.js'; +import { ZkWasmUtil, PaginationResult, QueryParams, Task, VerifyProofParams, AutoSubmitStatus, Round1Status, Round1Info } from "zkwasm-service-helper"; +import { U8ArrayUtil } from './lib.js'; import { submitRawProof} from "./prover.js"; import { decodeWithdraw} from "./convention.js"; import dotenv from 'dotenv'; @@ -20,18 +20,19 @@ const signer = new ethers.Wallet(get_settle_private_account(), provider); // const constants = { proxyAddress: get_contract_addr(), + chainId: get_chain_id(), }; function convertToBigUint64Array(combinedRoot: bigint): BigUint64Array { - const result = new BigUint64Array(4); + const result = new BigUint64Array(4); - for (let i = 3; i >= 0; i--) { + for (let i = 3; i >= 0; i--) { result[i] = combinedRoot & BigInt(2n ** 64n - 1n); combinedRoot = combinedRoot >> 64n; - } + } - return result; - } + return result; +} export async function getMerkleArray(): Promise{ // Connect to the Proxy contract @@ -77,15 +78,15 @@ const db = mongoose.connection; db.on('error', console.error.bind(console, 'connection error:')); db.once('open', function() {console.log("Connected to the database")}); -async function getTask(taskid: string, d_state: string|null) { +async function getTask(taskid: string, d_state: string|null): Promise { const queryParams: QueryParams = { - id: taskid, - tasktype: "Prove", - taskstatus: d_state, - user_address: null, - md5: get_image_md5(), - total: 1, - }; + id: taskid, + tasktype: "Prove", + taskstatus: d_state, + user_address: null, + md5: get_image_md5(), + total: 1, + }; const response: PaginationResult = await ServiceHelper.loadTasks( queryParams @@ -94,10 +95,15 @@ async function getTask(taskid: string, d_state: string|null) { return response.data[0]; } -async function getTaskWithTimeout(taskId: string, timeout: number): Promise { - return Promise.race([getTask(taskId, "Done"), new Promise((_, reject) => - setTimeout(() => reject(new Error('load proof task Timeout exceeded')), timeout)) - ]); +async function getTaskWithTimeout(taskId: string, timeout: number): Promise { + return Promise.race([ + getTask(taskId, "Done"), + new Promise( + (resolve:(v:null)=>void, reject) => setTimeout( + () => reject(new Error('load proof task Timeout exceeded')), timeout + ) + ) + ]); } export async function getWithdrawEventParameters( @@ -119,8 +125,8 @@ export async function getWithdrawEventParameters( if (decoded) { const l1token = decoded.args.l1token; const l1account = decoded.args.l1account; - const amount = decoded.args.amount; //in Wei - //console.log({ l1token, l1account, amount }); + const amount = decoded.args.amount; //in Wei + //console.log({ l1token, l1account, amount }); r.push({ token: l1token, address: l1account, @@ -132,90 +138,155 @@ export async function getWithdrawEventParameters( console.error("Log does not match event signature:", error); } }); - } catch (error) { console.error('Error retrieving withdraw event parameters:', error); } return r; } +/* We encode all params in string format of BN */ +interface ProofArgs { + txData: Uint8Array, + proofArr: Array, + verifyInstanceArr: Array, + auxArr: Array, + instArr: Array, +} + +async function prepareVerifyAttributesSingle(task: Task): Promise { + let shadowInstances = task.shadow_instances; + let batchInstances = task.batch_instances; + + let proofArr = new U8ArrayUtil(task.proof).toNumber(); + let auxArr = new U8ArrayUtil(task.aux).toNumber(); + let verifyInstancesArr = shadowInstances.length === 0 + ? new U8ArrayUtil(batchInstances).toNumber() + : new U8ArrayUtil(shadowInstances).toNumber(); + let instArr = new U8ArrayUtil(task.instances).toNumber(); + console.log("txData_orig:", task.input_context); + let txData = new Uint8Array(task.input_context); + console.log("txData:", txData); + console.log("txData.length:", txData.length); + return { + txData: txData, + proofArr: proofArr, + verifyInstanceArr: verifyInstancesArr, + auxArr: auxArr, + instArr: instArr, + } +} + +async function prepareVerifyAttributesBatch(task: Task): Promise { + let txData = new Uint8Array(task.input_context); + const round_1_info_response = await ServiceHelper.queryRound1Info({ + task_id: task._id.$oid, + chain_id: constants.chainId, + status: Round1Status.Batched, + total: 1, + }); + + let shadowInstances = task.shadow_instances; + let batchInstances = task.batch_instances; + + const round_1_output: Round1Info = round_1_info_response.data[0]; + + let verifyInstancesArr = shadowInstances.length === 0 + ? new U8ArrayUtil(batchInstances).toNumber() + : new U8ArrayUtil(shadowInstances).toNumber(); + + const siblingInstances = new U8ArrayUtil(new Uint8Array(round_1_output.target_instances[0])).toNumber(); + const r1ShadowInstance = new U8ArrayUtil(new Uint8Array(round_1_output.shadow_instances!)).toNumber()[0]; + + let instArr = new U8ArrayUtil(task.instances).toNumber(); + + // Find the index of this proof in the round 1 output by comparing task_ids + // This will be used to verify that this proof was included in a particular batch. + // If it does not exist, the verification will fail + const index = round_1_output.task_ids.findIndex( + (id:any) => id === task._id["$oid"] + ); + + let proofArr = siblingInstances; + proofArr.push(r1ShadowInstance); + + return { + txData: txData, + proofArr: proofArr, + verifyInstanceArr: verifyInstancesArr, + auxArr: [index.toString()], + instArr: instArr, + } +} + +async function prepareVerifyAttributes(task: Task): Promise { + if(task.proof_submit_mode == "Manual") { + return await prepareVerifyAttributesSingle(task); + } else { + return await prepareVerifyAttributesBatch(task); + } +} async function trySettle() { let merkleRoot = await getMerkle(); console.log("typeof :", typeof(merkleRoot[0])); console.log(merkleRoot); + const proxy = new ethers.Contract(constants.proxyAddress, abiData.abi, signer); + try { let record = await modelBundle.findOne({ merkleRoot: merkleRoot}); if (record) { let taskId = record.taskId; - let data0 = await getTaskWithTimeout(taskId, 60000); - - //check failed or just timeout - if (data0.proof.length == 0) { - let data1 = await getTask(taskId, null); - if (data1.status === "DryRunFailed" || data1.status === "Unprovable") { - console.log("Crash(Need manual review): task failed with state:", taskId, data1.status, data1.input_context); - while(1); //This is serious error, while loop to trigger manual review. - return -1; - } else { - console.log(`Task: ${taskId}, ${data1.status}, retry settle later.`); //will restart settle - return -1; - } + let task = await getTaskWithTimeout(taskId, 60000); + if (task!.proof_submit_mode == "Auto") { + const isRegistered = + task!.auto_submit_status === AutoSubmitStatus.RegisteredProof; + + if (!isRegistered) { + console.log("waiting for proof to be registered ... "); + return -1; + } } - let shadowInstances = data0.shadow_instances; - let batchInstances = data0.batch_instances; - - let proofArr = new U8ArrayUtil(data0.proof).toNumber(); - let auxArr = new U8ArrayUtil(data0.aux).toNumber(); - let verifyInstancesArr = shadowInstances.length === 0 - ? new U8ArrayUtil(batchInstances).toNumber() - : new U8ArrayUtil(shadowInstances).toNumber(); - let instArr = new U8ArrayUtil(data0.instances).toNumber(); - console.log("txData_orig:", data0.input_context); - let txData = new Uint8Array(data0.input_context); - console.log("txData:", txData); - console.log("txData.length:", txData.length); - - const proxy = new ethers.Contract(constants.proxyAddress, abiData.abi, signer); + + let attributes = await prepareVerifyAttributesSingle(task!); const tx = await proxy.verify( - txData, - proofArr, - verifyInstancesArr, - auxArr, - [instArr], + attributes.txData, + attributes.proofArr, + attributes.verifyInstanceArr, + attributes.auxArr, + [attributes.instArr], ); // wait for tx to be mined, can add no. of confirmations as arg const receipt = await tx.wait(); console.log("transaction:", tx.hash); console.log("receipt:", receipt); - const r = decodeWithdraw(txData); + const r = decodeWithdraw(attributes.txData); const s = await getWithdrawEventParameters(proxy, receipt); const withdrawArray = []; let status = 'Done'; if (r.length !== s.length) { - status = 'Fail'; - console.error("Arrays have different lengths,",r,s); + status = 'Fail'; + console.error("Arrays have different lengths,",r,s); } else { - for (let i = 0; i < r.length; i++) { - const rItem = r[i]; - const sItem = s[i]; - - if (rItem.address !== sItem.address || rItem.amount !== sItem.amount) { - console.log("Crash(Need manual review):"); - console.error(`Mismatch found: ${rItem.address}:${rItem.amount} ${sItem.address}:${sItem.amount}`); - while(1); //This is serious error, while loop to trigger manual review. - status = 'Fail'; - break; - } else { - // Assuming rItem is defined and has address and amount - record.withdrawArray.push({ - address: rItem.address, - amount: rItem.amount, - }); - } + for (let i = 0; i < r.length; i++) { + const rItem = r[i]; + const sItem = s[i]; + + if (rItem.address !== sItem.address || rItem.amount !== sItem.amount) { + console.log("Crash(Need manual review):"); + console.error(`Mismatch found: ${rItem.address}:${rItem.amount} ${sItem.address}:${sItem.amount}`); + while(1); //This is serious error, while loop to trigger manual review. + status = 'Fail'; + break; + } else { + // Assuming rItem is defined and has address and amount + record.withdrawArray.push({ + address: rItem.address, + amount: rItem.amount, + }); } + } } //update record record.settleTxHash = tx.hash; @@ -233,20 +304,20 @@ async function trySettle() { // start monitoring and settle async function main() { - while (true) { - try { - await trySettle(); - } catch (error) { - console.error("Error during trySettle:", error); - } - await new Promise(resolve => setTimeout(resolve, 60000)); - } + while (true) { + try { + await trySettle(); + } catch (error) { + console.error("Error during trySettle:", error); + } + await new Promise(resolve => setTimeout(resolve, 60000)); + } } // Check if this module is being run directly if (import.meta.url === `file://${process.argv[1]}`) { - console.log("Running settle.js directly"); - main().catch(console.error); + console.log("Running settle.js directly"); + main().catch(console.error); } else { - console.log("settle.js is being imported as a module"); + console.log("settle.js is being imported as a module"); }