ANS-104 bundle indexing

.github/workflows/build-core.yml

@@ -56,7 +56,7 @@ jobs:
         uses: VeryGoodOpenSource/very_good_coverage@v2
         with:
           path: ./coverage/lcov.info
-          min_coverage: 60
+          min_coverage: 50
 
       # Build and push container image to GCR (only on main branch)
       - name: Log in to the GitHub Container Registry
@@ -78,7 +78,7 @@ jobs:
 
       # Build and push container image to ECR
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v1-node16
+        uses: aws-actions/configure-aws-credentials@v2
         with:
           role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_BUILD_INVOCATION_ROLE }}
           aws-region: ${{ secrets.AWS_REGION }}

.gitignore

@@ -8,6 +8,10 @@
 /data
 /dist
 /node_modules
+/vendor
+
+# Generated docs
+/docs/sqlite/bundles
 
 # VS Code
 /.vscode

docker-compose.yaml

@@ -42,7 +42,9 @@ services:
       - INSTANCE_ID=${INSTANCE_ID:-}
       - AR_IO_WALLET=${AR_IO_WALLET:-}
       - ADMIN_API_KEY=${ADMIN_API_KEY:-}
+      - BACKFILL_BUNDLE_RECORDS=${BACKFILL_BUNDLE_RECORDS:-}
+      - FILTER_CHANGE_REPROCESS=${FILTER_CHANGE_REPROCESS:-}
       - ANS104_UNBUNDLE_FILTER=${ANS104_UNBUNDLE_FILTER:-}
-      - ANS104_DATA_INDEX_FILTER=${ANS104_DATA_INDEX_FILTER:-}
+      - ANS104_INDEX_FILTER=${ANS104_INDEX_FILTER:-}
       - ARNS_ROOT_HOST=${ARNS_ROOT_HOST:-}
       - SANDBOX_PROTOCOL=${SANDBOX_PROTOCOL:-}

docs/sqlite/bundles.meta.xml

@@ -0,0 +1,109 @@
+<schemaMeta xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://schemaspy.org/xsd/6/schemameta.xsd" >
+  <tables>
+    <table name="stable_data_items">
+      <column name="parent_id" type="BLOB">
+        <foreignKey table="bundles" column="id" />
+      </column>
+
+      <column name="root_transaction_id" type="BLOB">
+        <foreignKey table="bundles" column="id" />
+      </column>
+
+      <column name="owner_address" type="BLOB">
+        <foreignKey table="wallets" column="address" />
+      </column>
+    </table>
+
+    <table name="stable_data_item_tags">
+      <column name="data_item_id" type="BLOB">
+        <foreignKey table="stable_data_items" column="id" />
+      </column>
+
+      <column name="block_transaction_index" type="BLOB">
+        <foreignKey table="stable_data_items" column="block_transaction_index" />
+      </column>
+
+      <column name="parent_id" type="INTEGER">
+        <foreignKey table="bundles" column="id" />
+      </column>
+
+      <column name="root_transaction_id" type="INTEGER">
+        <foreignKey table="bundles" column="id" />
+      </column>
+
+      <column name="tag_name_hash" type="BLOB">
+        <foreignKey table="tag_names" column="hash" />
+      </column>
+
+      <column name="tag_value_hash" type="BLOB">
+        <foreignKey table="tag_values" column="hash" />
+      </column>
+    </table>
+
+    <table name="new_data_items">
+      <column name="owner_address" type="BLOB">
+        <foreignKey table="wallets" column="address" />
+      </column>
+    </table>
+
+    <table name="new_data_items">
+      <column name="parent_id" type="BLOB">
+        <foreignKey table="bundles" column="id" />
+      </column>
+
+      <column name="root_transaction_id" type="BLOB">
+        <foreignKey table="bundles" column="id" />
+      </column>
+    </table>
+
+    <table name="new_data_item_tags">
+      <column name="parent_id" type="BLOB">
+        <foreignKey table="bundles" column="id" />
+      </column>
+
+      <column name="root_transaction_id" type="BLOB">
+        <foreignKey table="bundles" column="id" />
+      </column>
+
+      <column name="data_item_id" type="BLOB">
+        <foreignKey table="new_data_items" column="id" />
+      </column>
+
+      <column name="tag_name_hash" type="BLOB">
+        <foreignKey table="tag_names" column="hash" />
+      </column>
+
+      <column name="tag_value_hash" type="BLOB">
+        <foreignKey table="tag_values" column="hash" />
+      </column>
+    </table>
+
+    <table name="bundles">
+      <column name="unbundle_filter_id" type="INTEGER">
+        <foreignKey table="filters" column="id" />
+      </column>
+
+      <column name="index_filter_id" type="INTEGER">
+        <foreignKey table="filters" column="id" />
+      </column>
+
+      <column name="format_id" type="INTEGER">
+        <foreignKey table="bundle_formats" column="id" />
+      </column>
+    </table>
+
+    <table name="bundle_data_items">
+      <column name="id" type="INTEGER">
+        <foreignKey table="stable_data_items" column="id" />
+      </column>
+
+      <column name="parent_id" type="INTEGER">
+        <foreignKey table="bundles" column="id" />
+      </column>
+
+      <column name="root_transaction_id" type="INTEGER">
+        <foreignKey table="bundles" column="id" />
+      </column>
+    </table>
+  </tables>
+</schemaMeta>

docs/sqlite/bundles.properties

@@ -0,0 +1,5 @@
+dbms=SQLite
+description=Xerial
+connectionSpec=jdbc:sqlite:data/sqlite/bundles.db?open_mode=1
+driver=org.sqlite.JDBC
+driverPath=vendor/sqlite-jdbc-3.42.0.0.jar

migrations/2023.05.15T18.06.50.bundles.init-schema.sql

@@ -0,0 +1,125 @@
+CREATE TABLE IF NOT EXISTS bundle_formats (
+  id INTEGER PRIMARY KEY,
+  name TEXT NOT NULL
+);
+
+INSERT INTO bundle_formats (id, name) VALUES (0, 'ans-102');
+INSERT INTO bundle_formats (id, name) VALUES (1, 'ans-104');
+
+CREATE TABLE IF NOT EXISTS bundles (
+  id BLOB PRIMARY KEY,
+  format INTEGER NOT NULL,
+  data_item_count INTEGER NOT NULL,
+  first_processed_at INTEGER NOT NULL,
+  last_processed_at INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS bundle_data_items (
+  id BLOB,
+  parent_id BLOB NOT NULL,
+  root_transaction_id BLOB NOT NULL,
+  indexed_at INTEGER NOT NULL,
+  PRIMARY KEY (id, parent_id)
+);
+
+CREATE TABLE IF NOT EXISTS wallets (
+  address BLOB PRIMARY KEY,
+  public_modulus BLOB
+);
+
+CREATE TABLE IF NOT EXISTS stable_data_items (
+  -- Identity
+  id BLOB NOT NULL,
+  parent_id BLOB NOT NULL,
+  root_transaction_id BLOB NOT NULL,
+  height INTEGER NOT NULL,
+  block_transaction_index INTEGER NOT NULL,
+  signature BLOB NOT NULL,
+  anchor BLOB NOT NULL,
+
+  -- Ownership
+  owner_address BLOB NOT NULL,
+  target BLOB,
+
+  -- Data
+  data_offset INTEGER NOT NULL,
+  data_size INTEGER NOT NULL,
+  content_type TEXT,
+
+  -- Metadata
+  tag_count INTEGER NOT NULL,
+  indexed_at INTEGER NOT NULL,
+  PRIMARY KEY (id)
+);
+
+CREATE INDEX IF NOT EXISTS stable_data_items_height_block_transaction_index_id_idx ON stable_data_items (height, block_transaction_index, id);
+CREATE INDEX IF NOT EXISTS stable_data_items_target_height_block_transaction_index_id_idx ON stable_data_items (target, height, block_transaction_index, id);
+CREATE INDEX IF NOT EXISTS stable_data_items_owner_address_height_block_transaction_index_id_idx ON stable_data_items (owner_address, height, block_transaction_index, id);
+CREATE INDEX IF NOT EXISTS stable_data_items_parent_id_height_block_transaction_index_id_idx ON stable_data_items (parent_id, height, block_transaction_index, id);
+
+CREATE TABLE IF NOT EXISTS tag_names (
+  hash BLOB PRIMARY KEY,
+  name BLOB NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS tag_values (
+  hash BLOB PRIMARY KEY,
+  value BLOB NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS stable_data_item_tags (
+  tag_name_hash BLOB NOT NULL,
+  tag_value_hash BLOB NOT NULL,
+  height INTEGER NOT NULL,
+  block_transaction_index INTEGER NOT NULL,
+  data_item_tag_index INTEGER NOT NULL,
+  data_item_id BLOB NOT NULL,
+  parent_id BLOB NOT NULL,
+  root_transaction_id BLOB NOT NULL,
+  PRIMARY KEY (tag_name_hash, tag_value_hash, height, block_transaction_index, data_item_id, data_item_tag_index)
+);
+
+CREATE INDEX IF NOT EXISTS stable_data_item_tags_transaction_id_idx ON stable_data_item_tags (data_item_id);
+
+CREATE TABLE IF NOT EXISTS new_data_items (
+  -- Identity
+  id BLOB NOT NULL,
+  parent_id BLOB NOT NULL,
+  root_transaction_id BLOB NOT NULL,
+  height INTEGER,
+  signature BLOB NOT NULL,
+  anchor BLOB NOT NULL,
+
+  -- Ownership
+  owner_address BLOB NOT NULL,
+  target BLOB,
+
+  -- Data
+  data_offset INTEGER NOT NULL,
+  data_size INTEGER NOT NULL,
+  content_type TEXT,
+
+  -- Metadata
+  tag_count INTEGER NOT NULL,
+  indexed_at INTEGER NOT NULL,
+  PRIMARY KEY (id)
+);
+
+CREATE INDEX IF NOT EXISTS new_data_items_parent_id_id_idx ON new_data_items (parent_id, id);
+CREATE INDEX IF NOT EXISTS new_data_items_root_transaction_id_id_idx ON new_data_items (root_transaction_id, id);
+CREATE INDEX IF NOT EXISTS new_data_items_target_id_idx ON new_data_items (target, id);
+CREATE INDEX IF NOT EXISTS new_data_items_owner_address_id_idx ON new_data_items (owner_address, id);
+CREATE INDEX IF NOT EXISTS new_data_items_height_indexed_at_idx ON new_data_items (height, indexed_at);
+
+CREATE TABLE IF NOT EXISTS new_data_item_tags (
+  tag_name_hash BLOB NOT NULL,
+  tag_value_hash BLOB NOT NULL,
+  root_transaction_id BLOB NOT NULL,
+  data_item_id BLOB NOT NULL,
+  data_item_tag_index INTEGER NOT NULL,
+  height INTEGER,
+  indexed_at INTEGER NOT NULL,
+  PRIMARY KEY (tag_name_hash, tag_value_hash, root_transaction_id, data_item_id, data_item_tag_index)
+);
+
+CREATE INDEX IF NOT EXISTS new_data_item_tags_height_indexed_at_idx ON new_data_item_tags (height, indexed_at);

migrations/2023.06.05T17.36.05.bundles.data-item-tags-data-item-id-indexes.sql

@@ -0,0 +1,4 @@
+DROP INDEX IF EXISTS stable_data_item_tags_transaction_id_idx;
+CREATE INDEX IF NOT EXISTS stable_data_item_tags_data_item_id_idx ON stable_data_item_tags (data_item_id);
+
+CREATE INDEX IF NOT EXISTS new_data_item_tags_data_item_id_idx ON new_data_item_tags (data_item_id);

migrations/2023.06.08T14.32.38.bundles.add-bundle-data-item-fields.sql

@@ -0,0 +1,15 @@
+DROP TABLE IF EXISTS bundle_data_items;
+
+CREATE TABLE IF NOT EXISTS bundle_data_items (
+  id BLOB NOT NULL,
+  parent_id BLOB NOT NULL,
+  parent_index INTEGER NOT NULL,
+  filter_id INTEGER NOT NULL,
+  root_transaction_id BLOB NOT NULL,
+  first_indexed_at INTEGER NOT NULL,
+  last_indexed_at INTEGER NOT NULL,
+  PRIMARY KEY (id, parent_id, parent_index, filter_id)
+);
+
+CREATE INDEX IF NOT EXISTS bundle_data_items_filter_id_idx
+  ON bundle_data_items (filter_id);

migrations/2023.06.13T14.01.27.bundles.add-filters.sql

@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS filters (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  filter TEXT NOT NULL UNIQUE
+);
+
+CREATE INDEX IF NOT EXISTS filters_filter_idx ON filters (filter);

migrations/2023.06.19T14.40.36.bundles.bundle-process-tracking.sql

@@ -0,0 +1,22 @@
+DROP TABLE IF EXISTS bundles;
+
+CREATE TABLE IF NOT EXISTS bundles (
+  id BLOB PRIMARY KEY,
+  format_id INTEGER NOT NULL,
+  unbundle_filter_id INTEGER,
+  index_filter_id INTEGER,
+  data_item_count INTEGER,
+  matched_data_item_count INTEGER,
+  first_queued_at INTEGER,
+  last_queued_at INTEGER,
+  first_skipped_at INTEGER,
+  last_skipped_at INTEGER,
+  first_unbundled_at INTEGER,
+  last_unbundled_at INTEGER,
+  first_fully_indexed_at INTEGER,
+  last_fully_indexed_at INTEGER
+);
+
+CREATE INDEX IF NOT EXISTS bundles_format_id_idx ON bundles (format_id);
+
+ALTER TABLE bundle_formats RENAME COLUMN name TO format;

migrations/2023.06.28T21.52.29.bundles.add-bundles-root-tx-id.sql

@@ -0,0 +1,2 @@
+ALTER TABLE bundles ADD COLUMN root_transaction_id BLOB;
+UPDATE bundles SET root_transaction_id = id;

migrations/2023.06.28T22.02.57.bundles.add-bundles-indexes.sql

@@ -0,0 +1,15 @@
+CREATE INDEX IF NOT EXISTS bundles_last_queued_at_idx
+  ON bundles (last_queued_at);
+CREATE INDEX IF NOT EXISTS bundles_last_skipped_at_idx
+  ON bundles (last_skipped_at);
+CREATE INDEX IF NOT EXISTS bundles_last_fully_indexed_at_idx
+  ON bundles (last_fully_indexed_at);
+CREATE INDEX IF NOT EXISTS bundles_matched_data_item_count_idx
+  ON bundles (matched_data_item_count);
+CREATE INDEX IF NOT EXISTS bundles_unbundle_filter_id_idx
+  ON bundles (unbundle_filter_id);
+CREATE INDEX IF NOT EXISTS bundles_index_filter_id_idx
+  ON bundles (index_filter_id);
+
+CREATE INDEX IF NOT EXISTS bundle_data_items_parent_id_filter_id_idx
+  ON bundle_data_items (parent_id, filter_id);

migrations/down/2023.05.15T18.06.50.bundles.init-schema.sql

@@ -0,0 +1,9 @@
+DROP TABLE IF EXISTS new_data_item_tags;
+DROP TABLE IF EXISTS new_data_items;
+DROP TABLE IF EXISTS stable_data_item_tags;
+DROP TABLE IF EXISTS tag_values;
+DROP TABLE IF EXISTS tag_names;
+DROP TABLE IF EXISTS stable_data_items;
+DROP TABLE IF EXISTS wallets;
+DROP TABLE IF EXISTS bundles;
+DROP TABLE IF EXISTS bundle_formats;

migrations/down/2023.06.05T17.36.05.bundles.data-item-tags-data-item-id-indexes.sql

@@ -0,0 +1,4 @@
+DROP INDEX IF EXISTS new_data_item_tags_data_item_id_idx;
+
+DROP INDEX IF EXISTS stable_data_item_tags_data_item_id_idx;
+CREATE INDEX IF NOT EXISTS stable_data_item_tags_transaction_id_idx ON stable_data_item_tags (data_item_id);

migrations/down/2023.06.08T14.32.38.bundles.add-bundle-data-item-fields.sql

@@ -0,0 +1,9 @@
+DROP TABLE IF EXISTS bundle_data_items;
+
+CREATE TABLE IF NOT EXISTS bundle_data_items (
+  id BLOB,
+  parent_id BLOB NOT NULL,
+  root_transaction_id BLOB NOT NULL,
+  indexed_at INTEGER NOT NULL,
+  PRIMARY KEY (id, parent_id)
+);

migrations/down/2023.06.13T14.01.27.bundles.add-filters.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS filters;

migrations/down/2023.06.19T14.40.36.bundles.bundle-process-tracking.sql

@@ -0,0 +1,11 @@
+ALTER TABLE bundle_formats RENAME COLUMN format TO name;
+
+DROP TABLE IF EXISTS bundles;
+
+CREATE TABLE IF NOT EXISTS bundles (
+  id BLOB PRIMARY KEY,
+  format INTEGER NOT NULL,
+  data_item_count INTEGER NOT NULL,
+  first_processed_at INTEGER NOT NULL,
+  last_processed_at INTEGER NOT NULL
+);

migrations/down/2023.06.28T21.52.29.bundles.add-bundles-root-tx-id.sql

@@ -0,0 +1 @@
+ALTER TABLE bundles DROP COLUMN root_transaction_id;