refactor(sqlite): use null parent and root TX ID for optimistic data …

…items PE-6284 For expedience, the initial optimistic indexing code used placeholder values for root TX ID and parent. While this mostly works, it's confusing and produces inaccurate GQL results without extra post processing (optimistic data item parents are wrong). This change modifies the DB to allow null parent, root transaction ids, and offsets. It also adjusts types to allow for this and introduces extra conditionals to ensure that bundle imports and data indexing for optimistically indexed data items are not attempted. This prevents errant optimistic data items from making their way into the stable indexes.
ar-io · Jun 17, 2024 · c43240d · c43240d
1 parent 5e944bc
commit c43240d
Show file tree

Hide file tree

Showing 11 changed files with 261 additions and 90 deletions.
diff --git a/.eslintrc b/.eslintrc
@@ -28,7 +28,7 @@
         "allowAny": true
       }
     ],
-    "eqeqeq": 2,
+    "eqeqeq": ["error", "smart"],
     "jest-formatting/padding-around-describe-blocks": 2,
     "jest-formatting/padding-around-test-blocks": 2,
     "header/header": [2, "./resources/license.header.js"],

diff --git a/migrations/2024.06.13T13.52.01.bundles.nullable-parent-and-root.sql b/migrations/2024.06.13T13.52.01.bundles.nullable-parent-and-root.sql
@@ -0,0 +1,47 @@
+DROP TABLE new_data_items;
+
+CREATE TABLE new_data_items (
+  -- Identity
+  id BLOB NOT NULL,
+  parent_id BLOB,
+  root_transaction_id BLOB,
+  height INTEGER,
+  signature BLOB NOT NULL,
+  anchor BLOB NOT NULL,
+
+  -- Ownership
+  owner_address BLOB NOT NULL,
+  target BLOB,
+
+  -- Data
+  data_offset INTEGER,
+  data_size INTEGER NOT NULL,
+  content_type TEXT,
+
+  -- Metadata
+  tag_count INTEGER NOT NULL,
+  indexed_at INTEGER NOT NULL,
+  PRIMARY KEY (id)
+);
+
+CREATE INDEX new_data_items_parent_id_id_idx ON new_data_items (parent_id, id);
+CREATE INDEX new_data_items_root_transaction_id_id_idx ON new_data_items (root_transaction_id, id);
+CREATE INDEX new_data_items_target_id_idx ON new_data_items (target, id);
+CREATE INDEX new_data_items_owner_address_id_idx ON new_data_items (owner_address, id);
+CREATE INDEX new_data_items_height_indexed_at_idx ON new_data_items (height, indexed_at);
+
+DROP TABLE new_data_item_tags;
+
+CREATE TABLE new_data_item_tags (
+  tag_name_hash BLOB NOT NULL,
+  tag_value_hash BLOB NOT NULL,
+  root_transaction_id BLOB,
+  data_item_id BLOB NOT NULL,
+  data_item_tag_index INTEGER NOT NULL,
+  height INTEGER,
+  indexed_at INTEGER NOT NULL,
+  PRIMARY KEY (tag_name_hash, tag_value_hash, root_transaction_id, data_item_id, data_item_tag_index)
+);
+
+CREATE INDEX new_data_item_tags_height_indexed_at_idx ON new_data_item_tags (height, indexed_at);
+CREATE INDEX new_data_item_tags_data_item_id_idx ON new_data_item_tags (data_item_id);
diff --git a/migrations/down/2024.06.13T13.52.01.bundles.nullable-parent-and-root.sql b/migrations/down/2024.06.13T13.52.01.bundles.nullable-parent-and-root.sql
@@ -0,0 +1,48 @@
+-- down migration
+DROP TABLE new_data_item_tags;
+
+CREATE TABLE new_data_item_tags (
+  tag_name_hash BLOB NOT NULL,
+  tag_value_hash BLOB NOT NULL,
+  root_transaction_id BLOB NOT NULL,
+  data_item_id BLOB NOT NULL,
+  data_item_tag_index INTEGER NOT NULL,
+  height INTEGER,
+  indexed_at INTEGER NOT NULL,
+  PRIMARY KEY (tag_name_hash, tag_value_hash, root_transaction_id, data_item_id, data_item_tag_index)
+);
+
+CREATE INDEX new_data_item_tags_height_indexed_at_idx ON new_data_item_tags (height, indexed_at);
+CREATE INDEX new_data_item_tags_data_item_id_idx ON new_data_item_tags (data_item_id);
+
+DROP TABLE new_data_items;
+
+CREATE TABLE new_data_items (
+  -- Identity
+  id BLOB NOT NULL,
+  parent_id BLOB NOT NULL,
+  root_transaction_id BLOB NOT NULL,
+  height INTEGER,
+  signature BLOB NOT NULL,
+  anchor BLOB NOT NULL,
+
+  -- Ownership
+  owner_address BLOB NOT NULL,
+  target BLOB,
+
+  -- Data
+  data_offset INTEGER NOT NULL,
+  data_size INTEGER NOT NULL,
+  content_type TEXT,
+
+  -- Metadata
+  tag_count INTEGER NOT NULL,
+  indexed_at INTEGER NOT NULL,
+  PRIMARY KEY (id)
+);
+
+CREATE INDEX new_data_items_parent_id_id_idx ON new_data_items (parent_id, id);
+CREATE INDEX new_data_items_root_transaction_id_id_idx ON new_data_items (root_transaction_id, id);
+CREATE INDEX new_data_items_target_id_idx ON new_data_items (target, id);
+CREATE INDEX new_data_items_owner_address_id_idx ON new_data_items (owner_address, id);
+CREATE INDEX new_data_items_height_indexed_at_idx ON new_data_items (height, indexed_at);
diff --git a/src/database/standalone-sqlite.ts b/src/database/standalone-sqlite.ts
@@ -241,7 +241,7 @@ export function dataItemToDbRows(item: NormalizedDataItem, height?: number) {
   const newDataItemTags = [] as {
     tag_name_hash: Buffer;
     tag_value_hash: Buffer;
-    root_transaction_id: Buffer;
+    root_transaction_id: Buffer | null;
     data_item_id: Buffer;
     data_item_tag_index: number;
     indexed_at: number;
@@ -268,7 +268,7 @@ export function dataItemToDbRows(item: NormalizedDataItem, height?: number) {
     newDataItemTags.push({
       tag_name_hash: tagNameHash,
       tag_value_hash: tagValueHash,
-      root_transaction_id: fromB64Url(item.root_tx_id),
+      root_transaction_id: item.root_tx_id ? fromB64Url(item.root_tx_id) : null,
       data_item_id: id,
       data_item_tag_index: dataItemTagIndex,
       indexed_at: currentUnixTimestamp(),
@@ -282,22 +282,29 @@ export function dataItemToDbRows(item: NormalizedDataItem, height?: number) {
 
   wallets.push({ address: ownerAddressBuffer, public_modulus: ownerBuffer });
 
-  const parentId = fromB64Url(item.parent_id);
-  const rootTxId = fromB64Url(item.root_tx_id);
+  const parentId = item.parent_id ? fromB64Url(item.parent_id) : null;
+  const rootTxId = item.root_tx_id ? fromB64Url(item.root_tx_id) : null;
 
-  return {
-    tagNames,
-    tagValues,
-    newDataItemTags,
-    wallets,
-    bundleDataItem: {
+  // We do not insert bundle data item rows for opimistically indexed data
+  // items
+  let bundleDataItem;
+  if (rootTxId) {
+    bundleDataItem = {
       id,
       parent_id: parentId,
       parent_index: item.parent_index,
       root_transaction_id: rootTxId,
       indexed_at: currentUnixTimestamp(),
       filter: item.filter,
-    },
+    };
+  }
+
+  return {
+    tagNames,
+    tagValues,
+    newDataItemTags,
+    wallets,
+    bundleDataItem,
     newDataItem: {
       id,
       parent_id: parentId,
@@ -519,10 +526,14 @@ export class StandaloneSqliteDatabaseWorker {
           this.stmts.bundles.insertOrIgnoreWallet.run(row);
         }
 
-        this.stmts.bundles.upsertBundleDataItem.run({
-          ...rows.bundleDataItem,
-          filter_id: this.getFilterId(rows.bundleDataItem.filter),
-        });
+        // We do not insert bundle data item rows for opimistically indexed
+        // data items
+        if (rows.bundleDataItem) {
+          this.stmts.bundles.upsertBundleDataItem.run({
+            ...rows.bundleDataItem,
+            filter_id: this.getFilterId(rows.bundleDataItem.filter),
+          });
+        }
 
         this.stmts.bundles.upsertNewDataItem.run({
           ...rows.newDataItem,
@@ -828,7 +839,7 @@ export class StandaloneSqliteDatabaseWorker {
   }
 
   saveDataItem(item: NormalizedDataItem) {
-    const rootTxId = fromB64Url(item.root_tx_id);
+    const rootTxId = item.root_tx_id ? fromB64Url(item.root_tx_id) : null;
     const maybeTxHeight = this.stmts.bundles.selectTransactionHeight.get({
       transaction_id: rootTxId,
     })?.height;

diff --git a/src/routes/ar-io.ts b/src/routes/ar-io.ts
@@ -182,12 +182,12 @@ arIoRouter.post(
           target: dataItemHeader.target ?? '',
           anchor: dataItemHeader.anchor ?? '',
           // These fields are not yet known, to be backfilled
-          parent_id: 'AA',
-          root_tx_id: 'AA',
-          index: 0,
-          parent_index: 0,
-          data_hash: '',
-          data_offset: 0,
+          parent_id: null,
+          root_tx_id: null,
+          index: null,
+          parent_index: null,
+          data_hash: null,
+          data_offset: null,
           filter: config.ANS104_INDEX_FILTER_STRING,
         });
       }

diff --git a/src/system.ts b/src/system.ts
@@ -353,6 +353,15 @@ const bundleDataImporter = new BundleDataImporter({
 
 async function queueBundle(item: NormalizedDataItem | PartialJsonTransaction) {
   try {
+    if ('root_tx_id' in item && item.root_tx_id === null) {
+      log.debug('Skipping download of optimistically indexed data item', {
+        id: item.id,
+        rootTxId: item.root_tx_id,
+        parentId: item.parent_id,
+      });
+      return;
+    }
+
     await db.saveBundle({
       id: item.id,
       rootTransactionId: 'root_tx_id' in item ? item.root_tx_id : item.id,
@@ -373,11 +382,11 @@ async function queueBundle(item: NormalizedDataItem | PartialJsonTransaction) {
       });
       bundleDataImporter.queueItem(
         {
+          ...item,
           index:
             'parent_index' in item && item.parent_index !== undefined
               ? item.parent_index
               : -1, // parent indexes are not needed for L1
-          ...item,
         },
         isPrioritized,
       );
@@ -390,8 +399,11 @@ async function queueBundle(item: NormalizedDataItem | PartialJsonTransaction) {
         skippedAt: currentUnixTimestamp(),
       });
     }
-  } catch (error) {
-    log.error('Error saving or queueing bundle', error);
+  } catch (error: any) {
+    log.error('Error saving or queueing bundle', {
+      message: error.message,
+      stack: error.stack,
+    });
   }
 }
 

diff --git a/src/types.d.ts b/src/types.d.ts
@@ -249,7 +249,7 @@ export interface NestedDataIndexWriter {
   }): Promise<void>;
 }
 
-export interface NormalizedDataItem {
+export interface NormalizedBundleDataItem {
   id: string;
   index: number;
   parent_id: string;
@@ -268,6 +268,29 @@ export interface NormalizedDataItem {
   content_type?: string;
 }
 
+export interface NormalizedOptimisticDataItem {
+  id: string;
+  index: null;
+  parent_id: null;
+  parent_index: null;
+  root_tx_id: null;
+  signature: string;
+  owner: string;
+  owner_address: string;
+  target: string;
+  anchor: string;
+  tags: B64uTag[];
+  data_offset: null;
+  data_size: number;
+  data_hash: null;
+  filter?: string;
+  content_type?: string;
+}
+
+type NormalizedDataItem =
+  | NormalizedBundleDataItem
+  | NormalizedOptimisticDataItem;
+
 interface GqlPageInfo {
   hasNextPage: boolean;
 }

diff --git a/src/workers/ans104-data-indexer.ts b/src/workers/ans104-data-indexer.ts
@@ -81,8 +81,8 @@ export class Ans104DataIndexer {
       id: item.id,
       parentId: item.parent_id,
       rootTxId: item.root_tx_id,
-      dataOffset: item?.data_offset,
-      dataSize: item?.data_size,
+      dataOffset: item.data_offset,
+      dataSize: item.data_size,
     });
 
     try {
@@ -91,23 +91,53 @@ export class Ans104DataIndexer {
         typeof item.data_size === 'number'
       ) {
         log.debug('Indexing data item data...');
-        await this.contiguousDataIndex.saveDataContentAttributes({
-          id: item.id,
-          hash: item.data_hash,
-          dataSize: item.data_size,
-          contentType: item.content_type,
-        });
-        await this.indexWriter.saveNestedDataId({
-          id: item.id,
-          parentId: item.parent_id,
-          dataOffset: item.data_offset,
-          dataSize: item.data_size,
-        });
-        await this.indexWriter.saveNestedDataHash({
-          hash: item.data_hash,
-          parentId: item.parent_id,
-          dataOffset: item.data_offset,
-        });
+        if (item.data_hash != null) {
+          if (item.data_size != null) {
+            await this.contiguousDataIndex.saveDataContentAttributes({
+              id: item.id,
+              hash: item.data_hash,
+              dataSize: item.data_size,
+              contentType: item.content_type,
+            });
+          } else {
+            log.warn(
+              'Skipping data item data content attributes indexing due to missing data size.',
+            );
+          }
+
+          // Index data hash to parent ID relationship
+          if (item.parent_id != null && item.data_offset != null) {
+            await this.indexWriter.saveNestedDataHash({
+              hash: item.data_hash,
+              parentId: item.parent_id,
+              dataOffset: item.data_offset,
+            });
+          } else {
+            log.warn(
+              'Skipping data item nested data indexing due to missing parent ID or data offset.',
+            );
+          }
+        } else {
+          log.warn('Skipping data item data indexing due to missing hash.');
+        }
+
+        // Index data offset and size for ID to parent ID relationship
+        if (
+          item.parent_id != null &&
+          item.data_offset != null &&
+          item.data_size != null
+        ) {
+          await this.indexWriter.saveNestedDataId({
+            id: item.id,
+            parentId: item.parent_id,
+            dataOffset: item.data_offset,
+            dataSize: item.data_size,
+          });
+        } else {
+          log.warn(
+            'Skipping data item parent ID indexing due to missing parent ID, data offset, or data size.',
+          );
+        }
         metrics.dataItemsIndexedCounter.inc();
         this.eventEmitter.emit(events.ANS104_DATA_ITEM_DATA_INDEXED, item);
         log.debug('Data item data indexed.');

diff --git a/src/workers/ans104-unbundler.ts b/src/workers/ans104-unbundler.ts
@@ -110,7 +110,7 @@ export class Ans104Unbundler {
     const log = this.log.child({ method: 'unbundle', id: item.id });
     try {
       let rootTxId: string | undefined;
-      if ('root_tx_id' in item) {
+      if ('root_tx_id' in item && item.root_tx_id !== null) {
         // Data item with root_tx_id
         rootTxId = item.root_tx_id;
       } else if ('last_tx' in item) {