feat: root_parent_offset and X-AR-IO-Data-Item-Data-Offset header

migrations/2024.10.30T16.11.15.bundles.add-root-parent-offset.sql

@@ -0,0 +1,2 @@
+ALTER TABLE new_data_items ADD COLUMN root_parent_offset INTEGER;
+ALTER TABLE stable_data_items ADD COLUMN root_parent_offset INTEGER;

migrations/down/2024.10.30T16.11.15.bundles.add-root-parent-offset.sql

@@ -0,0 +1,2 @@
+ALTER TABLE new_data_items DROP COLUMN root_parent_offset;
+ALTER TABLE stable_data_items DROP COLUMN root_parent_offset;

src/constants.ts

@@ -25,6 +25,7 @@ export const headerNames = {
   verified: 'X-AR-IO-Verified',
   cache: 'X-Cache',
   rootTransactionId: 'X-AR-IO-Root-Transaction-Id',
+  dataItemDataOffset: 'X-AR-IO-Data-Item-Data-Offset',
   arnsTtlSeconds: 'X-ArNS-TTL-Seconds',
   arnsResolvedId: 'X-ArNS-Resolved-Id',
   arnsProcessId: 'X-ArNS-Process-Id',

src/database/sql/bundles/import.sql

@@ -90,12 +90,14 @@ INSERT INTO new_data_items (
   id, parent_id, root_transaction_id, height, signature, anchor,
   owner_address, target, data_offset, data_size, content_type,
   tag_count, indexed_at, signature_type, offset, size, owner_offset,
-  owner_size, signature_offset, signature_size, content_encoding
+  owner_size, signature_offset, signature_size, content_encoding,
+  root_parent_offset
 ) VALUES (
   @id, @parent_id, @root_transaction_id, @height, @signature, @anchor,
   @owner_address, @target, @data_offset, @data_size, @content_type,
   @tag_count, @indexed_at, @signature_type, @offset, @size, @owner_offset,
-  @owner_size, @signature_offset, @signature_size, @content_encoding
+  @owner_size, @signature_offset, @signature_size, @content_encoding,
+  @root_parent_offset
 ) ON CONFLICT DO
 UPDATE SET
   height = IFNULL(@height, height),

src/database/sql/core/data-attributes.sql

@@ -1,20 +1,52 @@
 -- selectDataAttributes
 SELECT *
 FROM (
-  SELECT data_root, data_size, content_type, content_encoding, null AS root_transaction_id, true AS stable
+  SELECT
+    data_root,
+    data_size,
+    content_type,
+    content_encoding,
+    null AS root_transaction_id,
+    true AS stable,
+    null AS root_parent_offset,
+    null AS data_offset
   FROM stable_transactions
   WHERE id = @id
   UNION
-  SELECT data_root, data_size, content_type, content_encoding, null AS root_transaction_id, false AS stable
+  SELECT
+    data_root,
+    data_size,
+    content_type,
+    content_encoding,
+    null AS root_transaction_id,
+    false AS stable,
+    null AS root_parent_offset,
+    null AS data_offset
   FROM new_transactions
   WHERE id = @id
   UNION
-  SELECT null AS data_root, data_size, content_type, content_encoding, root_transaction_id, true AS stable
+  SELECT
+    null AS data_root,
+    data_size,
+    content_type,
+    content_encoding,
+    root_transaction_id,
+    true AS stable,
+    root_parent_offset,
+    data_offset
   FROM bundles.stable_data_items
   WHERE id = @id
   UNION
-  SELECT null AS data_root, data_size, content_type, content_encoding, root_transaction_id, false AS stable
+  SELECT
+    null AS data_root,
+    data_size,
+    content_type,
+    content_encoding,
+    root_transaction_id,
+    false AS stable,
+    root_parent_offset,
+    data_offset
   FROM bundles.new_data_items
   WHERE id = @id
 )
-LIMIT 1
+LIMIT 1;

src/database/standalone-sqlite.ts

@@ -340,6 +340,7 @@ export function dataItemToDbRows(item: NormalizedDataItem, height?: number) {
       owner_offset: item.owner_offset,
       owner_size: item.owner_size,
       parent_id: parentId,
+      root_parent_offset: item.root_parent_offset,
       root_transaction_id: rootTxId,
       signature: item.signature !== null ? fromB64Url(item.signature) : null,
       signature_offset: item.signature_offset,
@@ -994,6 +995,8 @@ export class StandaloneSqliteDatabaseWorker {
       contentEncoding: coreRow?.content_encoding,
       contentType,
       rootTransactionId,
+      rootParentOffset: coreRow?.root_parent_offset,
+      dataOffset: coreRow?.data_offset,
       isManifest: contentType === MANIFEST_CONTENT_TYPE,
       stable: coreRow?.stable === true,
       verified: dataRow?.verified === 1,

src/lib/ans-104.ts

@@ -67,6 +67,7 @@ export function normalizeAns104DataItem({
   filter,
   ans104DataItem,
   dataHash,
+  rootParentOffset,
 }: {
   rootTxId: string;
   parentId: string;
@@ -75,6 +76,7 @@ export function normalizeAns104DataItem({
   filter: string;
   ans104DataItem: DataItemInfo;
   dataHash: string;
+  rootParentOffset: number;
 }): NormalizedDataItem {
   let contentType: string | undefined;
   let contentEncoding: string | undefined;
@@ -121,6 +123,7 @@ export function normalizeAns104DataItem({
     owner_size: ans104DataItem.ownerSize,
     parent_id: parentId,
     parent_index: parentIndex,
+    root_parent_offset: rootParentOffset,
     root_tx_id: rootTxId,
     signature: ans104DataItem.signature,
     signature_offset: ans104DataItem.signatureOffset,
@@ -277,10 +280,12 @@ export class Ans104Parser {
     rootTxId,
     parentId,
     parentIndex,
+    rootParentOffset,
   }: {
     rootTxId: string;
     parentId: string;
     parentIndex: number;
+    rootParentOffset: number;
   }): Promise<void> {
     // eslint-disable-next-line no-async-promise-executor
     return new Promise(async (resolve, reject) => {
@@ -341,6 +346,7 @@ export class Ans104Parser {
                 parentId,
                 parentIndex,
                 bundlePath,
+                rootParentOffset,
               },
             });
             this.drainQueue();
@@ -412,7 +418,8 @@ if (!isMainThread) {
       process.exit(0);
     }
 
-    const { rootTxId, parentId, parentIndex, bundlePath } = message;
+    const { rootTxId, parentId, parentIndex, bundlePath, rootParentOffset } =
+      message;
     let stream: fs.ReadStream | undefined = undefined;
     try {
       stream = fs.createReadStream(bundlePath);
@@ -460,6 +467,7 @@ if (!isMainThread) {
           filter: workerData.dataItemIndexFilterString,
           ans104DataItem: dataItem,
           dataHash: dataItemHash,
+          rootParentOffset,
         });
 
         if (await filter.match(normalizedDataItem)) {

src/routes/ar-io.ts

@@ -219,6 +219,7 @@ arIoRouter.post(
             owner_size: null,
             parent_id: null,
             parent_index: null,
+            root_parent_offset: null,
             root_tx_id: null,
             signature_offset: null,
             signature_size: null,

src/routes/data/handlers.ts

@@ -124,6 +124,16 @@ const setDataHeaders = ({
     res.header(headerNames.rootTransactionId, dataAttributes.rootTransactionId);
   }
 
+  if (
+    dataAttributes?.rootParentOffset !== undefined &&
+    dataAttributes?.dataOffset !== undefined
+  ) {
+    res.header(
+      headerNames.dataItemDataOffset,
+      (dataAttributes.rootParentOffset + dataAttributes.dataOffset).toString(),
+    );
+  }
+
   setDigestStableVerifiedHeaders({ res, dataAttributes, data });
 };
 

src/types.d.ts

@@ -273,6 +273,7 @@ export interface NormalizedBundleDataItem {
   owner_size: number;
   parent_id: string;
   parent_index: number;
+  root_parent_offset: number;
   root_tx_id: string;
   signature: string | null;
   signature_offset: number;
@@ -300,6 +301,7 @@ export interface NormalizedOptimisticDataItem {
   owner_size: null;
   parent_id: null;
   parent_index: null;
+  root_parent_offset: null;
   root_tx_id: null;
   signature: string | null;
   signature_offset: null;
@@ -508,6 +510,8 @@ export interface ContiguousDataAttributes {
   contentEncoding?: string;
   contentType?: string;
   rootTransactionId?: string;
+  rootParentOffset?: number;
+  dataOffset?: number;
   isManifest: boolean;
   stable: boolean;
   verified: boolean;

src/workers/ans104-unbundler.ts

@@ -24,6 +24,7 @@ import { Ans104Parser } from '../lib/ans-104.js';
 import {
   ContiguousDataSource,
   ItemFilter,
+  NormalizedBundleDataItem,
   NormalizedDataItem,
   PartialJsonTransaction,
 } from '../types.js';
@@ -37,6 +38,12 @@ interface IndexProperty {
 export type UnbundleableItem = (NormalizedDataItem | PartialJsonTransaction) &
   IndexProperty;
 
+const isNormalizedBundleDataItem = (
+  item: UnbundleableItem,
+): item is NormalizedBundleDataItem =>
+  (item as NormalizedBundleDataItem).root_parent_offset !== undefined &&
+  (item as NormalizedBundleDataItem).data_offset !== undefined;
+
 export class Ans104Unbundler {
   // Dependencies
   private log: winston.Logger;
@@ -138,10 +145,22 @@ export class Ans104Unbundler {
       }
       if (await this.filter.match(item)) {
         log.info('Unbundling bundle...');
+        let rootParentOffset = 0;
+
+        if (
+          isNormalizedBundleDataItem(item) &&
+          rootTxId !== item.id &&
+          item.root_parent_offset !== null &&
+          item.data_offset !== null
+        ) {
+          rootParentOffset = item.root_parent_offset + item.data_offset;
+        }
+
         await this.ans104Parser.parseBundle({
           rootTxId,
           parentId: item.id,
           parentIndex: item.index,
+          rootParentOffset,
         });
         log.info('Bundle unbundled.');
       }

test/end-to-end/data.test.ts

@@ -333,6 +333,69 @@ describe('X-AR-IO-Root-Transaction-Id header', function () {
   });
 });
 
+describe('X-AR-IO-Data-Item-Data-Offset header', function () {
+  let compose: StartedDockerComposeEnvironment;
+  const bundle = '-H3KW7RKTXMg5Miq2jHx36OHSVsXBSYuE2kxgsFj6OQ';
+  const bdi = 'fLxHz2WbpNFL7x1HrOyUlsAVHYaKSyj6IqgCJlFuv9g';
+  const di = 'Dc-q5iChuRWcsjVBFstEqmLTx4SWkGZxcVO9OTEGjkQ';
+
+  before(async function () {
+    await rimraf(`${projectRootPath}/data/sqlite/*.db*`, { glob: true });
+
+    compose = await new DockerComposeEnvironment(
+      projectRootPath,
+      'docker-compose.yaml',
+    )
+      .withEnvironment({
+        START_HEIGHT: '1',
+        STOP_HEIGHT: '1',
+        GET_DATA_CIRCUIT_BREAKER_TIMEOUT_MS: '100000',
+        ANS104_UNBUNDLE_FILTER: '{"always": true}',
+        ANS104_INDEX_FILTER: '{"always": true}',
+        ADMIN_API_KEY: 'secret',
+      })
+      .withBuild()
+      .withWaitStrategy('core-1', Wait.forHttp('/ar-io/info', 4000))
+      .up(['core']);
+
+    await axios.post(
+      'http://localhost:4000/ar-io/admin/queue-bundle',
+      { id: bundle },
+      {
+        headers: {
+          'Content-Type': 'application/json',
+          Authorization: 'Bearer secret',
+        },
+      },
+    );
+
+    // hopefully enough time to unbundle it
+    await wait(10000);
+  });
+
+  after(async function () {
+    await compose.down();
+  });
+
+  it('Verifying header for L1 bundle', async function () {
+    const res = await axios.head(`http://localhost:4000/raw/${bundle}`);
+
+    assert.equal(res.headers['x-ar-io-data-item-data-offset'], undefined);
+  });
+
+  it('Verifying header for bundle data item', async function () {
+    const res = await axios.head(`http://localhost:4000/raw/${bdi}`);
+
+    assert.equal(res.headers['x-ar-io-data-item-data-offset'], '3072');
+  });
+
+  it('Verifying header for data item inside bundle data item', async function () {
+    const res = await axios.head(`http://localhost:4000/raw/${di}`);
+
+    assert.equal(res.headers['x-ar-io-data-item-data-offset'], '5783');
+  });
+});
+
 describe('X-AR-IO headers', function () {
   describe('with ARNS_ROOT_HOST', function () {
     let compose: StartedDockerComposeEnvironment;