handle proving keys in dedicated package (#872)

.github/workflows/turbo-ci.yml

@@ -113,7 +113,6 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
         with:
           targets: wasm32-unknown-unknown
-      - run: pnpm turbo download-keys --cache-dir=.turbo
       - run: pnpm turbo format-check:rust --cache-dir=.turbo
       - run: pnpm turbo lint:rust --cache-dir=.turbo
 
@@ -142,5 +141,4 @@ jobs:
         with:
           version: 'latest'
       - uses: browser-actions/setup-firefox@v1
-      - run: pnpm turbo download-keys --cache-dir=.turbo
       - run: pnpm turbo test:rust --cache-dir=.turbo

.gitignore

@@ -51,3 +51,6 @@ lerna-debug.log*
 *.sln
 *.sw?
 *.pem
+
+# large binary files
+*_pk.bin

.prettierignore

@@ -3,4 +3,4 @@
 pnpm-lock.yaml
 packages/wasm/crate
 packages/wasm/wasm
-apps/extension/bin
+*_pk.bin

apps/extension/package.json

@@ -5,7 +5,6 @@
   "description": "chrome-extension",
   "type": "module",
   "scripts": {
-    "download-keys": "tsx src/utils/download-proving-keys.ts",
     "dev": "webpack --watch --mode=development -d inline-source-map",
     "clean": "rm -rfv dist bin",
     "build": "webpack",
@@ -43,6 +42,7 @@
     "zustand": "^4.5.2"
   },
   "devDependencies": {
+    "@penumbra-zone/keys": "workspace:*",
     "@penumbra-zone/polyfills": "workspace:*",
     "@radix-ui/react-icons": "^1.3.0",
     "@types/firefox-webext-browser": "^120.0.3",
@@ -60,7 +60,6 @@
     "style-loader": "^3.3.4",
     "tailwindcss": "^3.4.1",
     "ts-loader": "^9.5.1",
-    "tsx": "^4.7.1",
     "vite-plugin-top-level-await": "^1.4.1",
     "vite-plugin-wasm": "^3.3.0",
     "webpack": "^5.91.0",

apps/extension/src/wasm-build-action.ts

@@ -1,11 +1,21 @@
 import {
+  Action,
   TransactionPlan,
   WitnessData,
 } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/transaction/v1/transaction_pb';
 import type { JsonValue } from '@bufbuild/protobuf';
 import type { ActionBuildRequest } from '@penumbra-zone/types/src/internal-msg/offscreen';
 import { FullViewingKey } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/keys/v1/keys_pb';
 
+import actionKeys from '@penumbra-zone/keys';
+const keyFileNames: Partial<Record<Exclude<Action['action']['case'], undefined>, URL>> =
+  Object.fromEntries(
+    Object.entries(actionKeys).map(([action, keyFile]) => [
+      action,
+      new URL('keys/' + keyFile, PRAX_ORIGIN),
+    ]),
+  );
+
 // necessary to propagate errors that occur in promises
 // see: https://stackoverflow.com/questions/39992417/how-to-bubble-a-web-worker-error-in-a-promise-via-worker-onerror
 self.addEventListener(
@@ -48,12 +58,15 @@ async function executeWorker(
   // Dynamically load wasm module
   const penumbraWasmModule = await import('@penumbra-zone/wasm/src/build');
 
+  const actionType = transactionPlan.actions[actionPlanIndex]!.action.case!;
+
   // Build action according to specification in `TransactionPlan`
   const action = await penumbraWasmModule.buildActionParallel(
     transactionPlan,
     witness,
     fullViewingKey,
     actionPlanIndex,
+    keyFileNames[actionType]!.href,
   );
 
   return action.toJson();

apps/extension/webpack.config.js

@@ -3,11 +3,14 @@ import path from 'path';
 import CopyPlugin from 'copy-webpack-plugin';
 import HtmlWebpackPlugin from 'html-webpack-plugin';
 import webpack from 'webpack';
+import url from 'url';
 
 // Loads default vars from `.env` file in this directory.  If you set
 // environment variables, you will override those defaults.
 dotenv.config();
 
+const keysPackage = path.dirname(url.fileURLToPath(import.meta.resolve('@penumbra-zone/keys')));
+
 const definitions = {
   // process.env.NODE_ENV is automatically provided by DefinePlugin
 
@@ -112,7 +115,13 @@ export default (env, argv) => {
       }),
       new webpack.DefinePlugin(definitions),
       new CopyPlugin({
-        patterns: ['public', { from: 'bin', to: 'bin' }],
+        patterns: [
+          'public',
+          {
+            from: path.join(keysPackage, 'keys', '*_pk.bin'),
+            to: 'keys/[name][ext]',
+          },
+        ],
       }),
       // html entry points
       new HtmlWebpackPlugin({

package.json

@@ -6,7 +6,6 @@
     "build": "turbo run build",
     "compile": "turbo run compile",
     "dev": "turbo run dev  --concurrency 20",
-    "download-keys": "turbo run download-keys",
     "lint": "turbo run lint -- --max-warnings=0",
     "lint:rust": "turbo run lint:rust",
     "lint:fix": "turbo run lint -- --fix",
@@ -19,7 +18,7 @@
     "playwright-install": "playwright install",
     "changeset": "changeset",
     "changeset:publish": "changeset publish",
-    "all-check": "pnpm install && pnpm compile && pnpm format-check && pnpm lint && pnpm test && pnpm download-keys && pnpm build && pnpm format-check:rust && pnpm lint:rust && pnpm test:rust"
+    "all-check": "pnpm install && pnpm compile && pnpm format-check && pnpm lint && pnpm test && pnpm build && pnpm format-check:rust && pnpm lint:rust && pnpm test:rust"
   },
   "dependencies": {
     "@buf/cosmos_ibc.bufbuild_es": "1.8.0-20240327103030-e2006674271c.2",
@@ -34,6 +33,7 @@
   },
   "devDependencies": {
     "@changesets/cli": "^2.27.1",
+    "@penumbra-zone/keys": "workspace:*",
     "@penumbra-zone/wasm": "workspace:*",
     "@storybook/react-vite": "8.0.4",
     "@turbo/gen": "^1.13.0",

packages/eslint-config-custom/index.js

@@ -16,7 +16,7 @@ module.exports = {
   rules: {
     'import/extensions': [
       'error',
-      'always',
+      'ignorePackages',
       {
         ts: 'never',
         tsx: 'never',

packages/keys/action-keys.json

@@ -0,0 +1,8 @@
+{
+  "delegatorVote": "delegator_vote_pk.bin",
+  "output": "output_pk.bin",
+  "spend": "spend_pk.bin",
+  "swap": "swap_pk.bin",
+  "swapClaim": "swapclaim_pk.bin",
+  "undelegateClaim": "convert_pk.bin"
+}

packages/keys/download-keys

@@ -0,0 +1,52 @@
+#!/bin/sh
+set -e
+defaultKeysVersion="v0.71.0"
+
+usage() {
+    echo "${1}"
+    echo "Usage: $(basename ${0}) <output-path> [<git tag> <sha256 manifest>]"
+    exit 64 # EX_USAGE
+}
+
+if [ -z "${PENUMBRA_KEYS_DIR}" ]; then
+    [ -z "${1}" ] && usage "No output path specified."
+    PENUMBRA_KEYS_DIR=$1
+fi
+
+if [ -z "${PENUMBRA_KEYS_VERSION}"]; then
+    if [ -z "${2}" ]; then
+        PENUMBRA_KEYS_VERSION=$defaultKeysVersion
+    else
+        [ -z "${3}" ] && usage "No sha256 provided."
+        PENUMBRA_KEYS_VERSION=$2
+        PENUMBRA_KEYS_SHA256=$(readlink -f "${3}")
+    fi
+fi
+
+if [ -z "${SKIP_DOWNLOAD_KEYS}" ]; then
+    keysUrl="https://github.com/penumbra-zone/penumbra/raw/${PENUMBRA_KEYS_VERSION}/crates/crypto/proof-params/src/gen/"
+    keysGlob="{convert,delegator_vote,nullifier_derivation,output,spend,swap,swapclaim}_pk.bin"
+    curl --output-dir "${PENUMBRA_KEYS_DIR}/${PENUMBRA_KEYS_VERSION}" --continue-at - \
+        --parallel --create-dirs --location --remote-name \
+        "${keysUrl}${keysGlob}"
+fi
+
+ln -vf "${PENUMBRA_KEYS_DIR}/${PENUMBRA_KEYS_VERSION}/"*_pk.bin "${PENUMBRA_KEYS_DIR}"
+
+shacmd=$(which sha256sum || which shasum)
+[ -z $shacmd ] && usage "No sha256sum or shasum tool available."
+
+cd "${PENUMBRA_KEYS_DIR}"
+if [ -f "${PENUMBRA_KEYS_SHA256}" ]; then
+    ${shacmd} -c "${PENUMBRA_KEYS_SHA256}"
+elif [ "${PENUMBRA_KEYS_VERSION}" = "${defaultKeysVersion}" ]; then
+    ${shacmd} -c <<EOF
+882527eb795af2819f8df4f6233d5844757768fb03d51a000f11b100eeea573f  convert_pk.bin
+6712a90216c7c0cfb12ef2a37d48ffae666f8d2bd9a722836864d0a70064dce7  delegator_vote_pk.bin
+ac9568335c1c158edddb2a70212c7b98d7c7aeff633b485a088049ecc01cc0ef  nullifier_derivation_pk.bin
+a8c5ccc9d6c74150a21a8411e90d523e6ebb0f2a2985327d9fb4ef690600461e  output_pk.bin
+69694b15abd96fdbeba5bbe1ba982d2240129be05813a9693acce31a1b43b0fd  spend_pk.bin
+3d75cac8a227cd8c6680cd40f94a436b89f10d9b9bb1a118cf4479f1aa9b3fef  swap_pk.bin
+8501f1dad9ac85d80c6421b17b838f8c6478431babfa836d3d33b5398fa6b6ad  swapclaim_pk.bin
+EOF
+fi

packages/keys/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@penumbra-zone/keys",
+  "version": "0.71.0",
+  "description": "Tool to download proving keys for Penumbra",
+  "type": "module",
+  "files": [
+    "action-keys.json",
+    "download-keys",
+    "keys"
+  ],
+  "exports": {
+    ".": "./action-keys.json",
+    "./*_pk.bin": "./keys/*_pk.bin"
+  },
+  "scripts": {
+    "postinstall": "./download-keys ./keys"
+  },
+  "bin": {
+    "penumbra-download-keys": "./download-keys"
+  }
+}

packages/wasm/.gitignore

@@ -1,3 +1,3 @@
 crate/target
 crate/**/*.rs.bk
-crate/wasm-pack.log
+crate/wasm-pack.log

packages/wasm/crate/src/keys.rs

@@ -27,10 +27,10 @@ pub fn load_proving_key(key: &[u8], key_type: &str) -> WasmResult<()> {
     let proving_key_map = match key_type {
         "spend" => &SPEND_PROOF_PROVING_KEY,
         "output" => &OUTPUT_PROOF_PROVING_KEY,
-        "delegator_vote" => &DELEGATOR_VOTE_PROOF_PROVING_KEY,
+        "delegatorVote" => &DELEGATOR_VOTE_PROOF_PROVING_KEY,
         "swap" => &SWAP_PROOF_PROVING_KEY,
-        "swapclaim" => &SWAPCLAIM_PROOF_PROVING_KEY,
-        "convert" => &CONVERT_PROOF_PROVING_KEY,
+        "swapClaim" => &SWAPCLAIM_PROOF_PROVING_KEY,
+        "undelegateClaim" => &CONVERT_PROOF_PROVING_KEY,
         _ => return Err(anyhow::anyhow!("Unsupported key type").into()),
     };
 

packages/wasm/crate/tests/build.rs

@@ -49,23 +49,20 @@ mod tests {
         // that are based on constructing objects according to protobuf definitions.
 
         // Load the proving key parameters as byte arrays.
-        let spend_key: &[u8] = include_bytes!("../../../../apps/extension/bin/spend_pk.bin");
-        let output_key: &[u8] = include_bytes!("../../../../apps/extension/bin/output_pk.bin");
-        let delegator_vote_key: &[u8] =
-            include_bytes!("../../../../apps/extension/bin/delegator_vote_pk.bin");
-        let swap_key: &[u8] = include_bytes!("../../../../apps/extension/bin/swap_pk.bin");
-        let swapclaim_key: &[u8] =
-            include_bytes!("../../../../apps/extension/bin/swapclaim_pk.bin");
-        let convert_key: &[u8] = include_bytes!("../../../../apps/extension/bin/convert_pk.bin");
+        let spend_key: &[u8] = include_bytes!("../../../keys/keys/spend_pk.bin");
+        let output_key: &[u8] = include_bytes!("../../../keys/keys/output_pk.bin");
+        let delegator_vote_key: &[u8] = include_bytes!("../../../keys/keys/delegator_vote_pk.bin");
+        let swap_key: &[u8] = include_bytes!("../../../keys/keys/swap_pk.bin");
+        let swapclaim_key: &[u8] = include_bytes!("../../../keys/keys/swapclaim_pk.bin");
+        let convert_key: &[u8] = include_bytes!("../../../keys/keys/convert_pk.bin");
 
         // Dynamically load the proving keys at runtime for each key type.
         load_proving_key(spend_key, "spend").expect("can load spend key");
         load_proving_key(output_key, "output").expect("can load output key");
-        load_proving_key(delegator_vote_key, "delegator_vote")
-            .expect("can load delegator vote key");
+        load_proving_key(delegator_vote_key, "delegatorVote").expect("can load delegator vote key");
         load_proving_key(swap_key, "swap").expect("can load swap key");
-        load_proving_key(swapclaim_key, "swapclaim").expect("can load swapclaim key");
-        load_proving_key(convert_key, "convert").expect("can load convert key");
+        load_proving_key(swapclaim_key, "swapClaim").expect("can load swapclaim key");
+        load_proving_key(convert_key, "undelegateClaim").expect("can load convert key");
 
         // Define database parameters.
         #[derive(Clone, Debug, Serialize, Deserialize)]

packages/wasm/package.json

@@ -16,12 +16,12 @@
     "test:rust": "cd crate ; wasm-pack test --headless --firefox -- --test build --target wasm32-unknown-unknown --release --features 'mock-database'"
   },
   "dependencies": {
+    "@penumbra-zone/keys": "workspace:*",
     "@penumbra-zone/types": "workspace:*",
     "zod": "^3.22.4"
   },
   "devDependencies": {
     "@penumbra-zone/bech32": "workspace:*",
-    "fake-indexeddb": "^5.0.2",
-    "tsx": "^4.7.1"
+    "fake-indexeddb": "^5.0.2"
   }
 }

packages/wasm/src/build.ts

@@ -7,7 +7,6 @@ import {
 } from '@buf/penumbra-zone_penumbra.bufbuild_es/penumbra/core/transaction/v1/transaction_pb';
 import type { StateCommitmentTree } from '@penumbra-zone/types/src/state-commitment-tree';
 import { authorize, build_action, build_parallel, load_proving_key, witness } from '../wasm';
-import { ActionType, provingKeys } from './proving-keys';
 import {
   FullViewingKey,
   SpendKey,
@@ -43,11 +42,12 @@ export const buildActionParallel = async (
   witnessData: WitnessData,
   fullViewingKey: FullViewingKey,
   actionId: number,
+  keyPath?: string,
 ): Promise<Action> => {
   // Conditionally read proving keys from disk and load keys into WASM binary
   const actionPlan = txPlan.actions[actionId];
   if (!actionPlan?.action.case) throw new Error('No action key provided');
-  await loadProvingKey(actionPlan.action.case);
+  if (keyPath) await loadProvingKey(actionPlan.action.case, keyPath);
 
   const result = build_action(
     txPlan.toBinary(),
@@ -59,12 +59,10 @@ export const buildActionParallel = async (
   return Action.fromBinary(result);
 };
 
-const loadProvingKey = async (actionType: ActionType) => {
-  const keyType = provingKeys[actionType];
-  if (!keyType) return;
-
-  const res = await fetch(`bin/${keyType}_pk.bin`);
-  const buffer = await res.arrayBuffer();
-  const uint8Array = new Uint8Array(buffer);
-  load_proving_key(uint8Array, keyType);
+const loadProvingKey = async (
+  actionType: Exclude<Action['action']['case'], undefined>,
+  keyPath: string,
+) => {
+  const key = new Uint8Array(await (await fetch(keyPath)).arrayBuffer());
+  load_proving_key(key, actionType);
 };