feat(cross-chain-indexing): enable multiple concurrent plugins (#24)

* feat(cross-subname-indexing): allow activating multiple plugins together

* feat(cross-subname-indexing): demonstrate retrieving cross-subname index

* refactor(handlers): stremline logic

* refactor(ponder.config): simplify structure

* fix: apply pr feedback

* fix: typos

* fix: apply pr feedback

* feat(plugins): introduce ACTIVE_PLUGINS validation

* fix(schema): use proper schema import

* feat(utils): import deep merge npm lib

* fix(codestyle): apply formatting

* docs: apply text edits

Apply PR suggestion regarding the in-code documentation

Co-authored-by: lightwalker.eth <[email protected]>

* feat(database-entites): ensure domain entity exists

Create an idepotenet function that takes a domain token ID and makes sure there is a database entity created for it.

* fix(upserts): make `ensureDomainExists` to always return a DB entity` -m "Had to replace `.onConflictDoNothing()` with `.onConflictDoUpdate({})` as the former would not return a DB entity if there was a conflict. The later would apply the delta on conflict, but since delta is an empty object, there will be no update at all. And yet, the existing DB entity will be always returned."

* docs: update description of `ensureDomainExists`

* fix(defaults): make all available plugins active

Declare all plugins active with the default `ACTIVE_PLUGINS` env var.

* feat(deps): update ponder

* docs(handlers): update Registry descriptions

* fix(handlers-base.eth): remove dead code

* feat(rpc): define env vars with rate limits

This change allows defining env vars with rate limtis for selected RPC nodes.

* docs: update descriptions as suggested by PR feedback

Co-authored-by: lightwalker.eth <[email protected]>

* docs: update .env.local.example

Organize env vars in logical groups. Also, update the public RPC URLs to be the dRPC ones.

* docs(helpers): define default values as named consts

Introduce `DEFAULT_RPC_RATE_LIMIT` const to describe a defult rate limit setting.

* refactor(db-helpers): rename `upserts into db-helpers

The new name describes the file contents better. There are some upsert helpers, but also some idemptent-insert ones.

* refactor(db-helpers): drop `ensureDomainExists`

Use plain DB inserts and apply conflict resolution method if applicatble.

* chore: trigger rebuild with fresh deployment id

* fix(readme): update title

* refactor(helpers): update `rpcRequestRateLimit` config facotry

* docs: typos

* docs(helpers): move the docs around

* refactor(ponder.config): renames

* docs(handlers): update registry setup description

* docs: replacements

* feat(helpers): introduce `rpcEndpointUrl` factory

* fix(codestyle): apply auto-formatting

* docs: update descriptions

Co-authored-by: lightwalker.eth <[email protected]>

* fix(helpers): update env var parsing logic

* docs: being more specific and correct

* fix(plugin-helpers): rename types

* fix: apply pr feedback

* docs: update env example

.env.local.example

@@ -1,19 +1,40 @@
 # RPC configuration
-# Follow the format: RPC_URL_{chainId}={rpcUrl}
-
-RPC_URL_1=https://ethereum-rpc.publicnode.com
-RPC_URL_8453=https://base-rpc.publicnode.com
-RPC_URL_59144=https://linea-rpc.publicnode.com
-
-# Identify which indexer plugin to activate (see `src/plugins` for available plugins)
-
-ACTIVE_PLUGIN=base.eth
+# For the RPC URL of each chain, follow the format: RPC_URL_{chainId}={rpcUrl}
+RPC_URL_1=https://eth.drpc.org
+RPC_URL_8453=https://base.drpc.org
+RPC_URL_59144=https://linea.drpc.org
+# For the RPC rate limits of each chain, follow the format:
+# RPC_REQUEST_RATE_LIMIT_{chainId}={rateLimitInRequestsPerSecond}
+# The rate limit is the maximum number of requests per second that can be made
+# to the RPC endpoint. For public RPC endpoints, it is recommended to set
+# a rate limit to low values (i.e. below 30 rps) to avoid being rate limited.
+# For private RPC endpoints, the rate limit can be set to higher values,
+# depending on the capacity of the endpoint. For example, 500 rps.
+# If no rate limit is set for a given chainId, the DEFAULT_RPC_RATE_LIMIT value
+# will be applied.
+RPC_REQUEST_RATE_LIMIT_1=50
+RPC_REQUEST_RATE_LIMIT_8453=20
+RPC_REQUEST_RATE_LIMIT_59144=20
 
 # Database configuration
-
-# This is where the indexer will create the tables defined in ponder.schema.ts
-# No two indexer instances can use the same database schema at the same time. This prevents data corruption.
-# @link https://ponder.sh/docs/api-reference/database#database-schema-rules
-DATABASE_SCHEMA=subname_index_base.eth
-# The indexer will use Postgres with that as the connection string. If not defined, the indexer will use PSlite.
+# This is a namespace for the tables that the indexer will create to store indexed data.
+# It should be a string that is unique to the running indexer instance.
+#
+# Keeping the database schema unique to the indexer instance is important to
+# 1) speed up indexing after a restart
+# 2) prevent data corruption from multiple indexer app instances writing state
+#    concurrently to the same db schema
+#
+# No two indexer instances can use the same database schema at the same time.
+#
+# Read more about database schema rules here:
+# https://ponder.sh/docs/api-reference/database#database-schema-rules
+DATABASE_SCHEMA=ens
+# This is the connection string for the database that the indexer will use to store data.
+# It should be in the format of `postgresql://<username>:<password>@<host>:<port>/<database>`
 DATABASE_URL=postgresql://dbuser:abcd1234@localhost:5432/my_database
+
+# Plugin configuration
+# Identify which indexer plugins to activate (see `src/plugins` for available plugins)
+# This is a comma separated list of one or more available plugin names (case-sensitive).
+ACTIVE_PLUGINS=eth,base.eth,linea.eth

package.json

@@ -14,7 +14,8 @@
   "dependencies": {
     "@ensdomains/ensjs": "^4.0.2",
     "hono": "^4.6.14",
-    "ponder": "^0.8.17",
+    "ponder": "^0.8.26",
+    "ts-deepmerge": "^7.0.2",
     "viem": "^2.21.57"
   },
   "devDependencies": {

ponder.config.ts

@@ -1,29 +1,33 @@
-import { ACTIVE_PLUGIN } from "./src/lib/plugin-helpers";
+import { deepMergeRecursive } from "./src/lib/helpers";
+import { type MergedTypes, getActivePlugins } from "./src/lib/plugin-helpers";
 import * as baseEthPlugin from "./src/plugins/base.eth/ponder.config";
 import * as ethPlugin from "./src/plugins/eth/ponder.config";
 import * as lineaEthPlugin from "./src/plugins/linea.eth/ponder.config";
 
-const plugins = [baseEthPlugin, ethPlugin, lineaEthPlugin] as const;
+// list of all available plugins
+// any available plugin can be activated at runtime
+const availablePlugins = [baseEthPlugin, ethPlugin, lineaEthPlugin] as const;
 
-// here we export only a single 'plugin's config, by type it as every config
-// this makes all of the mapping types happy at typecheck-time, but only the
-// relevant config is run at runtime
-export default ((): AllConfigs => {
-  const pluginToActivate = plugins.find((p) => p.ownedName === ACTIVE_PLUGIN);
+// merge of all available plugin configs to support correct typechecking
+// of the indexing handlers
+type AllPluginConfigs = MergedTypes<(typeof availablePlugins)[number]["config"]>;
 
-  if (!pluginToActivate) {
-    throw new Error(`Unsupported ACTIVE_PLUGIN: ${ACTIVE_PLUGIN}`);
-  }
+// Activates the indexing handlers of activated plugins.
+// Statically typed as the merge of all available plugin configs. However at
+// runtime returns the merge of all activated plugin configs.
+function activatePluginsAndGetConfig(): AllPluginConfigs {
+  const activePlugins = getActivePlugins(availablePlugins);
 
-  pluginToActivate.activate();
+  // load indexing handlers from the active plugins into the runtime
+  activePlugins.forEach((plugin) => plugin.activate());
 
-  return pluginToActivate.config as AllConfigs;
-})();
+  const activePluginsConfig = activePlugins
+    .map((plugin) => plugin.config)
+    .reduce((acc, val) => deepMergeRecursive(acc, val), {} as AllPluginConfigs);
 
-// Helper type to get the intersection of all config types
-type IntersectionOf<T> = (T extends any ? (x: T) => void : never) extends (x: infer R) => void
-  ? R
-  : never;
-// The type of the exported default is the intersection of all plugin configs to
-// each plugin can be correctly typechecked
-type AllConfigs = IntersectionOf<(typeof plugins)[number]["config"]>;
+  return activePluginsConfig as AllPluginConfigs;
+}
+
+// The type of the default export is a merge of all active plugin configs
+// configs so that each plugin can be correctly typechecked
+export default activatePluginsAndGetConfig();

src/handlers/NameWrapper.ts

@@ -2,10 +2,10 @@ import { type Context, type Event, type EventNames } from "ponder:registry";
 import schema from "ponder:schema";
 import { checkPccBurned } from "@ensdomains/ensjs/utils";
 import { type Address, type Hex, hexToBytes, namehash } from "viem";
+import { upsertAccount } from "../lib/db-helpers";
 import { bigintMax } from "../lib/helpers";
 import { makeEventId } from "../lib/ids";
 import { decodeDNSPacketBytes, tokenIdToLabel } from "../lib/subname-helpers";
-import { upsertAccount } from "../lib/upserts";
 
 // if the wrappedDomain in question has pcc burned (?) and a higher (?) expiry date, update the domain's expiryDate
 async function materializeDomainExpiryDate(context: Context, node: Hex) {

src/handlers/Registrar.ts

@@ -2,8 +2,8 @@ import { type Context } from "ponder:registry";
 import schema from "ponder:schema";
 import { Block } from "ponder";
 import { type Hex, namehash } from "viem";
+import { upsertAccount, upsertRegistration } from "../lib/db-helpers";
 import { isLabelIndexable, makeSubnodeNamehash, tokenIdToLabel } from "../lib/subname-helpers";
-import { upsertAccount, upsertRegistration } from "../lib/upserts";
 
 const GRACE_PERIOD_SECONDS = 7776000n; // 90 days in seconds
 
@@ -23,7 +23,7 @@ export const makeRegistrarHandlers = (ownedName: `${string}eth`) => {
     if (domain.labelName !== name) {
       await context.db
         .update(schema.domain, { id: node })
-        .set({ labelName: name, name: `${name}${ownedName}` });
+        .set({ labelName: name, name: `${name}.${ownedName}` });
     }
 
     await context.db.update(schema.registration, { id: label }).set({ labelName: name, cost });
@@ -79,7 +79,7 @@ export const makeRegistrarHandlers = (ownedName: `${string}eth`) => {
       context: Context;
       args: { name: string; label: Hex; cost: bigint };
     }) {
-      return await setNamePreimage(context, name, label, cost);
+      await setNamePreimage(context, name, label, cost);
     },
 
     async handleNameRenewedByController({
@@ -89,7 +89,7 @@ export const makeRegistrarHandlers = (ownedName: `${string}eth`) => {
       context: Context;
       args: { name: string; label: Hex; cost: bigint };
     }) {
-      return await setNamePreimage(context, name, label, cost);
+      await setNamePreimage(context, name, label, cost);
     },
 
     async handleNameRenewed({
@@ -117,7 +117,7 @@ export const makeRegistrarHandlers = (ownedName: `${string}eth`) => {
 
     async handleNameTransferred({
       context,
-      args: { tokenId, from, to },
+      args: { tokenId, to },
     }: {
       context: Context;
       args: {

src/handlers/Registry.ts

@@ -3,24 +3,47 @@ import schema from "ponder:schema";
 import { encodeLabelhash } from "@ensdomains/ensjs/utils";
 import { Block } from "ponder";
 import { type Hex, zeroAddress } from "viem";
+import { upsertAccount } from "../lib/db-helpers";
 import { makeResolverId } from "../lib/ids";
 import { ROOT_NODE, makeSubnodeNamehash } from "../lib/subname-helpers";
-import { upsertAccount } from "../lib/upserts";
 
 /**
  * Initialize the ENS root node with the zeroAddress as the owner.
+ * Any permutation of plugins might be activated (except no plugins activated)
+ * and multiple plugins expect the ENS root to exist. This behavior is
+ * consistent with the ens-subgraph, which initializes the ENS root node in
+ * the same way. However, the ens-subgraph does not have independent plugins.
+ * In our case, we have multiple plugins that might be activated independently.
+ * Regardless of the permutation of active plugins, they all expect
+ * the ENS root to exist.
+ *
+ * This function should be used as the setup event handler for registry
+ * (or shadow registry) contracts.
+ * https://ponder.sh/docs/api-reference/indexing-functions#setup-event
+ * In case there are multiple plugins activated, `setupRootNode` will be
+ * executed multiple times. The order of execution of `setupRootNode` is
+ * deterministic based on the reverse order of the network names of the given
+ * contracts associated with the activated plugins. For example,
+ * if the network names were: `base`, `linea`, `mainnet`, the order of execution
+ * will be: `mainnet`, `linea`, `base`.
+ * And if the network names were: `base`, `ethereum`, `linea`, the order of
+ * execution will be: `linea`, `ethereum`, `base`.
  */
 export async function setupRootNode({ context }: { context: Context }) {
   // ensure we have an account for the zeroAddress
   await upsertAccount(context, zeroAddress);
 
   // initialize the ENS root to be owned by the zeroAddress and not migrated
-  await context.db.insert(schema.domain).values({
-    id: ROOT_NODE,
-    ownerId: zeroAddress,
-    createdAt: 0n,
-    isMigrated: false,
-  });
+  await context.db
+    .insert(schema.domain)
+    .values({
+      id: ROOT_NODE,
+      ownerId: zeroAddress,
+      createdAt: 0n,
+      isMigrated: false,
+    })
+    // only insert the domain entity into the database if it doesn't already exist
+    .onConflictDoNothing();
 }
 
 function isDomainEmpty(domain: typeof schema.domain.$inferSelect) {

src/handlers/Resolver.ts

@@ -2,9 +2,9 @@ import { type Context } from "ponder:registry";
 import schema from "ponder:schema";
 import { Log } from "ponder";
 import { Hex } from "viem";
+import { upsertAccount, upsertResolver } from "../lib/db-helpers";
 import { hasNullByte, uniq } from "../lib/helpers";
 import { makeResolverId } from "../lib/ids";
-import { upsertAccount, upsertResolver } from "../lib/upserts";
 
 export async function handleAddrChanged({
   context,

src/lib/upserts.ts → src/lib/db-helpers.ts

@@ -3,19 +3,19 @@ import schema from "ponder:schema";
 import type { Address } from "viem";
 
 export async function upsertAccount(context: Context, address: Address) {
-  return await context.db.insert(schema.account).values({ id: address }).onConflictDoNothing();
+  return context.db.insert(schema.account).values({ id: address }).onConflictDoNothing();
 }
 
 export async function upsertResolver(
   context: Context,
   values: typeof schema.resolver.$inferInsert,
 ) {
-  return await context.db.insert(schema.resolver).values(values).onConflictDoUpdate(values);
+  return context.db.insert(schema.resolver).values(values).onConflictDoUpdate(values);
 }
 
 export async function upsertRegistration(
   context: Context,
   values: typeof schema.registration.$inferInsert,
 ) {
-  return await context.db.insert(schema.registration).values(values).onConflictDoUpdate(values);
+  return context.db.insert(schema.registration).values(values).onConflictDoUpdate(values);
 }