Merge pull request #132 from penumbra-zone/paginate-suspense-ibc-tables

Paginate IBC tables and add support for prefetching and `Suspense`

src/app/api/blocks/route.ts

@@ -20,14 +20,13 @@ export async function POST(req: Request) {
       SELECT b.height, b.created_at FROM blocks b
       ORDER BY b.height DESC LIMIT $queryLimit! OFFSET $pageOffset!;
     `;
-    const getBlocksCount = sql<IGetBlocksCountQuery>`SELECT COUNT(*)::int as _count FROM blocks;`;
+    const getBlocksCount = sql<IGetBlocksCountQuery>`SELECT COUNT(*)::int as "count!" FROM blocks;`;
 
     console.log("Acquiring DB Client and Querying database for recent blocks.");
 
     const client = await getPgClient();
     const blocks = await getBlocksByDesc.run({queryLimit, pageOffset}, client);
-    const [{ _count },,] = await getBlocksCount.run(undefined, client);
-    const count = _count ?? 1;
+    const [{ count },,] = await getBlocksCount.run(undefined, client);
 
     console.log("Successfully queried Blocks.");
     console.log([count, blocks]);

src/app/api/blocks/route.types.ts

@@ -23,7 +23,7 @@ export type IGetBlocksCountParams = void;
 
 /** 'GetBlocksCount' return type */
 export interface IGetBlocksCountResult {
-  _count: number | null;
+  count: number;
 }
 
 /** 'GetBlocksCount' query type */

src/app/api/ibc/channels/route.ts

@@ -1,66 +1,80 @@
 import { getPgClient } from "@/lib/db";
 import { sql } from "@pgtyped/runtime";
-import { IGetIbcChannelsQuery } from "./route.types";
+import { IGetChannelsCountQuery, IGetIbcChannelsQuery } from "./route.types";
+import { NextRequest } from "next/server";
 
-export async function GET() {
+export async function GET(req: NextRequest) {
   console.log("SUCCESS: GET /api/ibc/channels");
   try {
+    const pageParam = req.nextUrl.searchParams.get("page")?.trim() ?? "";
+    const pageOffset = (parseInt(pageParam, 10)) * 10;
+    const pageLimit = 10;
+
     console.log("Acquiring PgClient and querying for IBC channels");
 
     const client = await getPgClient();
 
-    // TODO: A lot to improve on here. Create stable views instead of CTE's per client request. Laterals not a bad idea, either.
-    // NOTE: The only suq-query that I'm not 100% on how to improve later is for getting the most up-to-date consensusHeight for a counterparty
-    //       in type_consensus_by_client.
+    // TODO: Revisit with latteral joins like other IBC queries
+    // WARNING
+    // TODO: This code *MUST* be revisited once support for IBC chanClose* events is clarified. Only channels are ever possibly deleted but does Penumbra support this?.
     const getIbcChannels = sql<IGetIbcChannelsQuery>`
-    WITH channel_connection (channel_id, connection_id) AS (
-      SELECT ea.value as channel_id, _ea.value as connection_id
-      FROM event_attributes ea
-      INNER JOIN event_attributes _ea
-        ON _ea.block_id=ea.block_id
-        AND _ea.composite_key='channel_open_init.connection_id'
-      WHERE
-        ea.composite_key='channel_open_init.channel_id'
-    ), connections_counterparty_by_client (client_id, connection_id, counterparty_client_id) AS (
-      SELECT ea.value as client_id, c_ea.value as connection_id, p_ea.value as counterparty_client_id
-      FROM event_attributes ea
-      INNER JOIN event_attributes c_ea
-        ON ea.block_id=c_ea.block_id
-        AND c_ea.composite_key='connection_open_init.connection_id'
-      INNER JOIN event_attributes p_ea
-        ON p_ea.block_id=c_ea.block_id
-        AND p_ea.composite_key='connection_open_init.counterparty_client_id'
-      WHERE
-        ea.composite_key='connection_open_init.client_id'
-    ), type_consensus_by_client (client_id, client_type, consensus_height) AS (
-      SELECT DISTINCT ON (updates.client_id, updates.client_type)
-        updates.client_id, updates.client_type, updates.consensus_height
-      FROM (
-        SELECT ea.block_id, ea.value as client_id, t_ea.value as client_type, h_ea.value as consensus_height
+      WITH channel_connection (channel_id, connection_id) AS (
+        SELECT ea.value as channel_id, _ea.value as connection_id
         FROM event_attributes ea
-        INNER JOIN event_attributes h_ea
-          ON h_ea.block_id=ea.block_id
-          AND h_ea.composite_key='update_client.consensus_height'
-        INNER JOIN event_attributes t_ea
-          ON t_ea.block_id=ea.block_id
-          AND t_ea.composite_key='update_client.client_type'
+        INNER JOIN event_attributes _ea
+          ON _ea.block_id=ea.block_id
+          AND _ea.composite_key='channel_open_init.connection_id'
         WHERE
-          ea.composite_key='update_client.client_id'
-        ORDER BY ea.block_id DESC
-      ) updates
-    )
-    SELECT cc.channel_id, tcc.client_id, ccc.connection_id, tcc.client_type, ccc.counterparty_client_id, tcc.consensus_height
-    FROM channel_connection cc
-    LEFT JOIN connections_counterparty_by_client ccc ON cc.connection_id=ccc.connection_id
-    LEFT JOIN type_consensus_by_client tcc ON tcc.client_id=ccc.client_id;
-    `;
+          ea.composite_key='channel_open_init.channel_id'
+      ), connections_counterparty_by_client (client_id, connection_id, counterparty_client_id) AS (
+        SELECT ea.value as client_id, c_ea.value as connection_id, p_ea.value as counterparty_client_id
+        FROM event_attributes ea
+        INNER JOIN event_attributes c_ea
+          ON ea.block_id=c_ea.block_id
+          AND c_ea.composite_key='connection_open_init.connection_id'
+        INNER JOIN event_attributes p_ea
+          ON p_ea.block_id=c_ea.block_id
+          AND p_ea.composite_key='connection_open_init.counterparty_client_id'
+        WHERE
+          ea.composite_key='connection_open_init.client_id'
+      ), type_consensus_by_client (client_id, client_type, consensus_height) AS (
+        SELECT DISTINCT ON (updates.client_id, updates.client_type)
+          updates.client_id, updates.client_type, updates.consensus_height
+        FROM (
+          SELECT ea.block_id, ea.value as client_id, t_ea.value as client_type, h_ea.value as consensus_height
+          FROM event_attributes ea
+          INNER JOIN event_attributes h_ea
+            ON h_ea.block_id=ea.block_id
+            AND h_ea.composite_key='update_client.consensus_height'
+          INNER JOIN event_attributes t_ea
+            ON t_ea.block_id=ea.block_id
+            AND t_ea.composite_key='update_client.client_type'
+          WHERE
+            ea.composite_key='update_client.client_id'
+          ORDER BY ea.block_id DESC
+        ) updates
+      )
+      SELECT cc.channel_id, tcc.client_id, ccc.connection_id, tcc.client_type, ccc.counterparty_client_id, tcc.consensus_height
+      FROM channel_connection cc
+      LEFT JOIN connections_counterparty_by_client ccc ON cc.connection_id=ccc.connection_id
+      LEFT JOIN type_consensus_by_client tcc ON tcc.client_id=ccc.client_id
+      LIMIT $pageLimit! OFFSET $pageOffset!
+    ;`;
+    const getChannelsCount = sql<IGetChannelsCountQuery>`
+      SELECT COUNT(*)::int as "count!"
+      FROM event_attributes ea
+      WHERE ea.composite_key='channel_open_init.channel_id'
+    ;`;
 
-    const channels = await getIbcChannels.run(undefined, client);
+    const channels = await getIbcChannels.run({ pageLimit, pageOffset }, client);
+    const [{ count },,] = await getChannelsCount.run(undefined, client);
     client.release();
 
-    console.log("Successfully queried channels:", channels);
+    console.log("Successfully queried channels:", [channels, count]);
+
+    const pages = Math.floor((count / 10) + 1);
 
-    return new Response(JSON.stringify(channels));
+    return new Response(JSON.stringify({ results: channels, pages }));
 
   } catch (error) {
     console.error("GET request failed.", error);

src/app/api/ibc/channels/route.types.ts

@@ -1,7 +1,10 @@
 /** Types generated for queries found in "src/app/api/ibc/channels/route.ts" */
 
 /** 'GetIbcChannels' parameters type */
-export type IGetIbcChannelsParams = void;
+export interface IGetIbcChannelsParams {
+  pageLimit: bigint | number;
+  pageOffset: bigint | number;
+}
 
 /** 'GetIbcChannels' return type */
 export interface IGetIbcChannelsResult {
@@ -19,3 +22,17 @@ export interface IGetIbcChannelsQuery {
   result: IGetIbcChannelsResult;
 }
 
+/** 'GetChannelsCount' parameters type */
+export type IGetChannelsCountParams = void;
+
+/** 'GetChannelsCount' return type */
+export interface IGetChannelsCountResult {
+  count: number;
+}
+
+/** 'GetChannelsCount' query type */
+export interface IGetChannelsCountQuery {
+  params: IGetChannelsCountParams;
+  result: IGetChannelsCountResult;
+}
+

src/app/api/ibc/clients/route.ts

@@ -1,11 +1,17 @@
 import { getPgClient } from "@/lib/db";
 import { sql } from "@pgtyped/runtime";
+import { NextRequest } from "next/server";
+import { IGetClientsQuery } from "./route.types";
 
-export async function GET() {
+export async function GET(req: NextRequest) {
   console.log("SUCCESS: GET /api/ibc/clients");
   try {
+    const pageParam = req.nextUrl.searchParams.get("page")?.trim() ?? "";
+    const pageOffset = (parseInt(pageParam, 10)) * 10;
+    const pageLimit = 10;
+
     console.log("Querying indexer for IBC clients.");
-    const getClients = sql`
+    const getClients = sql<IGetClientsQuery>`
       SELECT
         clients.client_id as "client_id!",
         clients.block_id as "block_id!",
@@ -18,11 +24,10 @@ export async function GET() {
         SELECT DISTINCT ON (ea.value) ea.value as "client_id", ea.block_id, ea.tx_id
         FROM event_attributes ea
         WHERE
-          ea.composite_key='create_client.client_id'
-          OR
           ea.composite_key='update_client.client_id'
+          OR
+          ea.composite_key='create_client.client_id'
         ORDER BY client_id, ea.block_id DESC
-        LIMIT 10
       ) clients LEFT JOIN LATERAL (
         SELECT MAX(ea.value) as "max_value", ea.tx_id
         FROM event_attributes ea
@@ -44,15 +49,32 @@ export async function GET() {
         LIMIT 1
       ) hashes ON true
       ORDER BY clients.block_id DESC
+      LIMIT $pageLimit! OFFSET $pageOffset!
+    ;`;
+    // Filtering distinct and then counting is more efficient than both at the same time. Not that these queries are particularly performant to begin with.
+    const getClientsCount = sql`
+      SELECT COUNT(*)::int as "count!"
+      FROM (
+        SELECT DISTINCT ON (ea.value) ea.value, ea.block_id
+        FROM event_attributes ea
+        WHERE
+          ea.composite_key='update_client.client_id'
+          OR
+          ea.composite_key='create_client.client_id'
+        ORDER BY ea.value, ea.block_id DESC
+      ) AS clients
     ;`;
+
     const pgClient = await getPgClient();
-    const clients = await getClients.run(undefined, pgClient);
+    const clients = await getClients.run({ pageLimit, pageOffset }, pgClient);
+    const [{ count },,] = await getClientsCount.run(undefined, pgClient);
     pgClient.release();
 
     console.log("Successfully queried for IBC Clients.");
-    // console.dir(["pgClient query: ", clients], { depth: 4 });
 
-    return new Response(JSON.stringify(clients));
+    const pages = Math.floor((count / 10) + 1);
+
+    return new Response(JSON.stringify({ results: clients, pages }));
 
   } catch (error) {
     console.error("GET request failed.", error);

src/app/api/ibc/clients/route.types.ts

@@ -1,7 +1,10 @@
 /** Types generated for queries found in "src/app/api/ibc/clients/route.ts" */
 
 /** 'GetClients' parameters type */
-export type IGetClientsParams = void;
+export interface IGetClientsParams {
+  pageLimit: bigint | number;
+  pageOffset: bigint | number;
+}
 
 /** 'GetClients' return type */
 export interface IGetClientsResult {
@@ -18,3 +21,17 @@ export interface IGetClientsQuery {
   result: IGetClientsResult;
 }
 
+/** 'GetClientsCount' parameters type */
+export type IGetClientsCountParams = void;
+
+/** 'GetClientsCount' return type */
+export interface IGetClientsCountResult {
+  count: number;
+}
+
+/** 'GetClientsCount' query type */
+export interface IGetClientsCountQuery {
+  params: IGetClientsCountParams;
+  result: IGetClientsCountResult;
+}
+

src/app/api/ibc/connections/route.ts

@@ -1,26 +1,40 @@
 import { getPgClient } from "@/lib/db";
 import { sql } from "@pgtyped/runtime";
-import { IGetConnectionsQuery } from "./route.types";
+import { IGetConnectionsCountQuery, IGetConnectionsQuery } from "./route.types";
+import { NextRequest } from "next/server";
 
-export async function GET() {
+export async function GET(req: NextRequest) {
   console.log("SUCCESS: GET /api/ibc/connections");
   try {
+    const pageParam = req.nextUrl.searchParams.get("page")?.trim() ?? "";
+    const pageOffset = (parseInt(pageParam, 10)) * 10;
+    const pageLimit = 10;
+
     console.log("Acquiring db and querying for IBC Connections.");
 
     const client = await getPgClient();
+
     const getConnections = sql<IGetConnectionsQuery>`
       SELECT ea.value as "connection_id!"
       FROM event_attributes ea
       WHERE ea.composite_key='connection_open_init.connection_id'
-      ORDER BY ea.block_id DESC;
-    `;
+      ORDER BY ea.block_id DESC LIMIT $pageLimit OFFSET $pageOffset!
+    ;`;
+    const getConnectionsCount = sql<IGetConnectionsCountQuery>`
+      SELECT COUNT(*)::int as "count!"
+      FROM event_attributes ea
+      WHERE ea.composite_key='connection_open_init.connection_id'
+    ;`;
 
-    const connections = await getConnections.run(undefined, client);
+    const connections = await getConnections.run({ pageOffset, pageLimit }, client);
+    const [{ count },, ] = await getConnectionsCount.run(undefined, client);
     client.release();
 
-    console.log("Successfully queried:", connections);
+    console.log("Successfully queried:", [connections, count]);
+
+    const pages = Math.floor((count / 10) + 1);
 
-    return new Response(JSON.stringify(connections));
+    return new Response(JSON.stringify({ results: connections, pages }));
 
   } catch (error) {
     console.error("GET request failed.", error);

src/app/api/ibc/connections/route.types.ts

@@ -1,7 +1,10 @@
 /** Types generated for queries found in "src/app/api/ibc/connections/route.ts" */
 
 /** 'GetConnections' parameters type */
-export type IGetConnectionsParams = void;
+export interface IGetConnectionsParams {
+  pageLimit?: bigint | number | null | void;
+  pageOffset: bigint | number;
+}
 
 /** 'GetConnections' return type */
 export interface IGetConnectionsResult {
@@ -14,3 +17,17 @@ export interface IGetConnectionsQuery {
   result: IGetConnectionsResult;
 }
 
+/** 'GetConnectionsCount' parameters type */
+export type IGetConnectionsCountParams = void;
+
+/** 'GetConnectionsCount' return type */
+export interface IGetConnectionsCountResult {
+  count: number;
+}
+
+/** 'GetConnectionsCount' query type */
+export interface IGetConnectionsCountQuery {
+  params: IGetConnectionsCountParams;
+  result: IGetConnectionsCountResult;
+}
+