Jorge/10574 read fully

a3p-integration/proposals/z:acceptance/test-lib/batchQuery.js

@@ -0,0 +1,176 @@
+import { assert } from '@endo/errors';
+import { E } from '@endo/far';
+
+/** @typedef {'children' | 'data'} AgoricChainStoragePathKind */
+/** @template T @typedef {import('@endo/marshal').FromCapData<T>} FromCapData<T> */
+/** @template T @typedef {import('@endo/eventual-send').ERef<T>} ERef<T> */
+
+/**
+ * @param {[kind: AgoricChainStoragePathKind, item: string]} path
+ */
+export const pathToKey = path => path.join('.');
+
+/** @param {string} key */
+export const keyToPath = key => {
+  const [kind, ...rest] = key.split('.');
+  assert(kind === 'children' || kind === 'data');
+  /** @type {[kind: 'children' | 'data', item: string]} */
+  const out = [kind, rest.join('.')];
+  return out;
+};
+
+/**
+ * @template T
+ * @param {(value: string) => T} f
+ * @param {AsyncGenerator<string[], void, unknown>} chunks
+ */
+async function* mapHistory(f, chunks) {
+  for await (const chunk of chunks) {
+    if (chunk === undefined) continue;
+    for (const value of chunk.reverse()) {
+      yield f(value);
+    }
+  }
+}
+
+/**
+ * @param {ERef<import('./makeHttpClient').LCD>} lcd
+ */
+export const makeVStorage = lcd => {
+  const getJSON = (href, options) => E(lcd).getJSON(href, options);
+
+  // height=0 is the same as omitting height and implies the highest block
+  const href = (path = 'published', { kind = 'data' } = {}) =>
+    `/agoric/vstorage/${kind}/${path}`;
+  const headers = height =>
+    height ? { 'x-cosmos-block-height': `${height}` } : undefined;
+
+  const readStorage = (
+    path = 'published',
+    { kind = 'data', height = 0 } = {},
+  ) =>
+    getJSON(href(path, { kind }), { headers: headers(height) }).catch(err => {
+      throw Error(`cannot read ${kind} of ${path}: ${err.message}`);
+    });
+  const readCell = (path, opts) =>
+    readStorage(path, opts)
+      .then(data => data.value)
+      .then(s => (s === '' ? {} : JSON.parse(s)));
+
+  /**
+   * Read values going back as far as available
+   *
+   * @param {string} path
+   * @param {number | string} [minHeight]
+   */
+  async function* readHistory(path, minHeight = undefined) {
+    // undefined the first iteration, to query at the highest
+    let blockHeight;
+    await null;
+    do {
+      console.debug('READING', { blockHeight });
+      /** @type {string[]} */
+      let values = [];
+      try {
+        ({ blockHeight, values } = await readCell(path, {
+          kind: 'data',
+          height: blockHeight && Number(blockHeight) - 1,
+        }));
+        console.debug('readAt returned', { blockHeight });
+      } catch (err) {
+        if (err.message.match(/unknown request/)) {
+          // XXX FIXME
+          console.error(err);
+          break;
+        }
+        throw err;
+      }
+      yield values;
+      console.debug('PUSHED', values);
+      console.debug('NEW', { blockHeight, minHeight });
+      if (minHeight && Number(blockHeight) <= Number(minHeight)) break;
+    } while (blockHeight > 0);
+  }
+
+  /**
+   * @template T
+   * @param {(value: string) => T} f
+   * @param {string} path
+   * @param {number | string} [minHeight]
+   */
+  const readHistoryBy = (f, path, minHeight) =>
+    mapHistory(f, readHistory(path, minHeight));
+
+  return {
+    lcd,
+    readStorage,
+    readCell,
+    readHistory,
+    readHistoryBy,
+  };
+};
+
+/** @typedef {ReturnType<typeof makeVStorage>} VStorage */
+
+/** @param {string | unknown} d */
+const parseIfJSON = d => {
+  if (typeof d !== 'string') return d;
+  try {
+    return JSON.parse(d);
+  } catch {
+    return d;
+  }
+};
+
+/**
+ * @param {ReturnType<makeVStorage>} vstorage
+ * @param {FromCapData<string>} unmarshal
+ * @param {[AgoricChainStoragePathKind, string][]} paths
+ */
+export const batchVstorageQuery = async (vstorage, unmarshal, paths) => {
+  const requests = paths.map(([kind, path]) =>
+    vstorage.readStorage(path, { kind }),
+  );
+
+  return Promise.all(requests).then(responses =>
+    responses.map((res, index) => {
+      if (paths[index][0] === 'children') {
+        return [
+          pathToKey(paths[index]),
+          { value: res.children, blockHeight: undefined },
+        ];
+      }
+
+      if (!res.value) {
+        return [
+          pathToKey(paths[index]),
+          {
+            error: `Cannot parse value of response for path [${
+              paths[index]
+            }]: ${JSON.stringify(res)}`,
+          },
+        ];
+      }
+
+      const data = parseIfJSON(res.value);
+
+      const latestValue =
+        typeof data.values !== 'undefined'
+          ? parseIfJSON(data.values[data.values.length - 1])
+          : parseIfJSON(data.value);
+
+      const unserialized =
+        typeof latestValue.slots !== 'undefined'
+          ? unmarshal(latestValue)
+          : latestValue;
+
+      return [
+        pathToKey(paths[index]),
+        {
+          blockHeight: data.blockHeight,
+          value: unserialized,
+        },
+      ];
+    }),
+  );
+};

a3p-integration/proposals/z:acceptance/test-lib/makeHttpClient.js

@@ -0,0 +1,100 @@
+import { assert } from '@endo/errors';
+import { Far } from '@endo/far';
+
+const { freeze } = Object;
+
+const jsonType = { 'Content-Type': 'application/json' };
+
+const filterBadStatus = res => {
+  if (res.status >= 400) {
+    throw Error(`Bad status on response: ${res.status}`);
+  }
+  return res;
+};
+
+/**
+ * Make an RpcClient using explicit access to the network.
+ *
+ * The RpcClient implementations included in cosmjs
+ * such as {@link https://cosmos.github.io/cosmjs/latest/tendermint-rpc/classes/HttpClient.html HttpClient}
+ * use ambient authority (fetch or axios) for network access.
+ *
+ * To facilitate cooperation without vulnerability,
+ * as well as unit testing, etc. this RpcClient maker takes
+ * network access as a parameter, following
+ * {@link https://github.com/Agoric/agoric-sdk/wiki/OCap-Discipline|OCap Discipline}.
+ *
+ * @param {string} url
+ * @param {typeof globalThis.fetch} fetch
+ * @returns {import('@cosmjs/tendermint-rpc').RpcClient}
+ */
+export const makeHttpClient = (url, fetch) => {
+  const headers = {}; // XXX needed?
+
+  // based on cosmjs 0.30.1:
+  // https://github.com/cosmos/cosmjs/blob/33271bc51cdc865cadb647a1b7ab55d873637f39/packages/tendermint-rpc/src/rpcclients/http.ts#L37
+  // https://github.com/cosmos/cosmjs/blob/33271bc51cdc865cadb647a1b7ab55d873637f39/packages/tendermint-rpc/src/rpcclients/httpclient.ts#L25
+  return freeze({
+    disconnect: () => {
+      // nothing to be done
+    },
+
+    /**
+     * @param {import('@cosmjs/json-rpc').JsonRpcRequest} request
+     */
+    execute: async request => {
+      const settings = {
+        method: 'POST',
+        body: request ? JSON.stringify(request) : undefined,
+        headers: { ...jsonType, ...headers },
+      };
+      return fetch(url, settings)
+        .then(filterBadStatus)
+        .then(res => res.json());
+    },
+  });
+};
+
+/**
+ * gRPC-gateway REST API access
+ *
+ * @see {@link https://docs.cosmos.network/v0.45/core/grpc_rest.html#rest-server Cosmos SDK REST Server}
+ *
+ * Note: avoid Legacy REST routes, per
+ * {@link https://docs.cosmos.network/v0.45/migrations/rest.html Cosmos SDK REST Endpoints Migration}.
+ *
+ * @param {string} apiAddress nodes default to port 1317
+ * @param {object} io
+ * @param {typeof fetch} io.fetch
+ */
+export const makeAPI = (apiAddress, { fetch }) => {
+  assert.typeof(apiAddress, 'string');
+
+  /**
+   * @param {string} href
+   * @param {object} [options]
+   * @param {Record<string, string>} [options.headers]
+   */
+  const getJSON = (href, options = {}) => {
+    const opts = {
+      keepalive: true,
+      headers: {
+        'Content-Type': 'application/json',
+        ...options.headers,
+      },
+    };
+    const url = `${apiAddress}${href}`;
+    return fetch(url, opts).then(r => {
+      if (!r.ok) throw Error(r.statusText);
+      return r.json().then(data => {
+        return data;
+      });
+    });
+  };
+
+  return Far('LCD', {
+    getJSON,
+    latestBlock: () => getJSON(`/cosmos/base/tendermint/v1beta1/blocks/latest`),
+  });
+};
+/** @typedef {ReturnType<typeof makeAPI>} LCD */

a3p-integration/proposals/z:acceptance/test.sh

@@ -5,32 +5,38 @@ set -ueo pipefail
 # The effects of this step are not persisted in further proposal layers.
 
 # test the state right after the previous proposals
-yarn ava initial.test.js
+# yarn ava initial.test.js
 
-# XXX some of these tests have path dependencies so no globs
-yarn ava core-eval.test.js
+# # XXX some of these tests have path dependencies so no globs
+# yarn ava core-eval.test.js
 
-npm install -g tsx
-scripts/test-vaults.mts
+# npm install -g tsx
+# scripts/test-vaults.mts
 
-echo ACCEPTANCE TESTING kread
-yarn ava kread.test.js
+# echo ACCEPTANCE TESTING kread
+# yarn ava kread.test.js
 
-echo ACCEPTANCE TESTING valueVow
-yarn ava valueVow.test.js
+# echo ACCEPTANCE TESTING valueVow
+# yarn ava valueVow.test.js
+
+echo ACCEPTANCE TESTING passing vstorage
+yarn ava vstorage.test.js
 
 echo ACCEPTANCE TESTING state sync
 ./state-sync-snapshots-test.sh
 ./genesis-test.sh
 
-echo ACCEPTANCE TESTING wallet
-yarn ava wallet.test.js
+echo ACCEPTANCE TESTING failing vstorage
+yarn ava vstorage.test.js
+
+# echo ACCEPTANCE TESTING wallet
+# yarn ava wallet.test.js
 
-echo ACCEPTANCE TESTING psm
-yarn ava psm.test.js
+# echo ACCEPTANCE TESTING psm
+# yarn ava psm.test.js
 
-echo ACCEPTANCE TESTING governance
-yarn ava governance.test.js
+# echo ACCEPTANCE TESTING governance
+# yarn ava governance.test.js
 
-echo ACCEPTANCE TESTING vaults
-yarn ava vaults.test.js
+# echo ACCEPTANCE TESTING vaults
+# yarn ava vaults.test.js

a3p-integration/proposals/z:acceptance/vstorage.test.js

@@ -0,0 +1,44 @@
+/* global fetch */
+
+import test from 'ava';
+import { makeVStorage } from '@agoric/client-utils';
+import { networkConfig } from './test-lib/index.js';
+import { makeVStorage as mockVstorage } from './test-lib/batchQuery.js';
+import { makeAPI } from './test-lib/makeHttpClient.js';
+
+test.skip('readFully should vstorage node history', async t => {
+  const vstorage = makeVStorage({ fetch }, networkConfig);
+  const { readLatest, readAt, readFully } = vstorage;
+
+  // this test was executed with different nodes and the same behavior was observed
+  const nodePath = 'published.committees.Economic_Committee.latestQuestion';
+
+  console.log('readLatest: ', await readLatest(nodePath));
+  console.log('readAt: ', await readAt(nodePath));
+
+  // Return a rejected promise
+  console.log('readFully: ', await readFully(nodePath));
+
+  t.pass();
+});
+
+test('readHistory should return vstorage node history', async t => {
+    const nodePath = 'published.committees.Economic_Committee.latestQuestion';
+    const apiAddress = 'http://0.0.0.0:1317';
+
+    const lcd = makeAPI(apiAddress, { fetch });
+    const { readHistory } = mockVstorage(lcd);
+
+    const historyIterator = await readHistory(nodePath);
+    const history = [];
+
+    for await (const data of historyIterator) {
+      if (data) {
+        history.push(data);
+      }
+    }
+    console.log('history: ', history);
+
+    t.true(history.length > 0);
+  });
+