feat: index and serve content-encoding on txs and data items

migrations/2024.07.22T18.33.32.core.add-tx-content-encoding.sql

@@ -0,0 +1,2 @@
+ALTER TABLE new_transactions ADD COLUMN content_encoding TEXT;
+ALTER TABLE stable_transactions ADD COLUMN content_encoding TEXT;

migrations/2024.07.22T18.33.45.bundles.add-data-item-content-encoding.sql

@@ -0,0 +1,2 @@
+ALTER TABLE new_data_items ADD COLUMN content_encoding TEXT;
+ALTER TABLE stable_data_items ADD COLUMN content_encoding TEXT;

migrations/down/2024.07.22T18.33.32.core.add-tx-content-encoding.sql

@@ -0,0 +1,2 @@
+ALTER TABLE new_transactions DROP COLUMN content_encoding;
+ALTER TABLE stable_transactions DROP COLUMN content_encoding;

migrations/down/2024.07.22T18.33.45.bundles.add-data-item-content-encoding.sql

@@ -0,0 +1,2 @@
+ALTER TABLE new_data_items DROP COLUMN content_encoding;
+ALTER TABLE stable_data_items DROP COLUMN content_encoding;

src/database/sql/bundles/flush.sql

@@ -6,15 +6,15 @@ INSERT INTO stable_data_items (
   data_offset, data_size, content_type,
   tag_count, indexed_at, signature_type,
   offset, size, owner_offset, owner_size,
-  signature_offset, signature_size
+  signature_offset, signature_size, content_encoding
 ) SELECT
   ndi.id, ndi.parent_id, ndi.root_transaction_id,
   ndi.height, sbt.block_transaction_index,
   ndi.signature, ndi.anchor, ndi.owner_address, ndi.target,
   ndi.data_offset, ndi.data_size, ndi.content_type,
   ndi.tag_count, ndi.indexed_at, ndi.signature_type,
   ndi.offset, ndi.size, ndi.owner_offset, ndi.owner_size,
-  ndi.signature_offset, ndi.signature_size
+  ndi.signature_offset, ndi.signature_size, ndi.content_encoding
 FROM new_data_items ndi
 JOIN core.stable_block_transactions sbt
   ON ndi.root_transaction_id = sbt.transaction_id

src/database/sql/bundles/import.sql

@@ -90,12 +90,12 @@ INSERT INTO new_data_items (
   id, parent_id, root_transaction_id, height, signature, anchor,
   owner_address, target, data_offset, data_size, content_type,
   tag_count, indexed_at, signature_type, offset, size, owner_offset,
-  owner_size, signature_offset, signature_size
+  owner_size, signature_offset, signature_size, content_encoding
 ) VALUES (
   @id, @parent_id, @root_transaction_id, @height, @signature, @anchor,
   @owner_address, @target, @data_offset, @data_size, @content_type,
   @tag_count, @indexed_at, @signature_type, @offset, @size, @owner_offset,
-  @owner_size, @signature_offset, @signature_size
+  @owner_size, @signature_offset, @signature_size, @content_encoding
 ) ON CONFLICT DO
 UPDATE SET
   height = IFNULL(@height, height),

src/database/sql/core/data-attributes.sql

@@ -1,19 +1,19 @@
 -- selectDataAttributes
 SELECT *
 FROM (
-  SELECT data_root, data_size, content_type, true AS stable
+  SELECT data_root, data_size, content_type, content_encoding, true AS stable
   FROM stable_transactions
   WHERE id = @id
   UNION
-  SELECT data_root, data_size, content_type, false AS stable
+  SELECT data_root, data_size, content_type, content_encoding, false AS stable
   FROM new_transactions
   WHERE id = @id
   UNION
-  SELECT null, data_size, content_type, true AS stable
+  SELECT null, data_size, content_type, content_encoding, true AS stable
   FROM bundles.stable_data_items
   WHERE id = @id
   UNION
-  SELECT null, data_size, content_type, false AS stable
+  SELECT null, data_size, content_type, content_encoding, false AS stable
   FROM bundles.new_data_items
   WHERE id = @id
 )

src/database/sql/core/flush.sql

@@ -36,11 +36,13 @@ ON CONFLICT DO NOTHING
 INSERT INTO stable_transactions (
   id, height, block_transaction_index, signature,
   format, last_tx, owner_address, target, quantity,
-  reward, data_size, data_root, content_type, tag_count
+  reward, data_size, data_root, content_type, tag_count,
+  content_encoding
 ) SELECT
   nt.id, nbt.height, nbt.block_transaction_index, nt.signature,
   nt.format, nt.last_tx, nt.owner_address, nt.target, nt.quantity,
-  nt.reward, nt.data_size, nt.data_root, nt.content_type, nt.tag_count
+  nt.reward, nt.data_size, nt.data_root, nt.content_type, nt.tag_count,
+  nt.content_encoding
 FROM new_transactions nt
 JOIN new_block_transactions nbt ON nbt.transaction_id = nt.id
 WHERE nbt.height < @end_height

src/database/sql/core/import.sql

@@ -72,11 +72,11 @@ INSERT INTO new_transaction_tags (
 INSERT INTO new_transactions (
   id, signature, format, last_tx, owner_address,
   target, quantity, reward, data_size, data_root,
-  tag_count, content_type, indexed_at, height
+  tag_count, content_type, indexed_at, height, content_encoding
 ) VALUES (
   @id, @signature, @format, @last_tx, @owner_address,
   @target, @quantity, @reward, @data_size, @data_root,
-  @tag_count, @content_type, @indexed_at, @height
+  @tag_count, @content_type, @indexed_at, @height, @content_encoding
 ) ON CONFLICT DO UPDATE SET height = IFNULL(@height, height)
 
 -- insertOrIgnoreMissingTransaction

src/database/standalone-sqlite.ts

@@ -163,6 +163,10 @@ function isContentTypeTag(tagName: Buffer) {
   return tagName.toString('utf8').toLowerCase() === 'content-type';
 }
 
+function isContentEncodingTag(tagName: Buffer) {
+  return tagName.toString('utf8').toLowerCase() === 'content-encoding';
+}
+
 function ownerToAddress(owner: Buffer) {
   return crypto.createHash('sha256').update(owner).digest();
 }
@@ -180,6 +184,8 @@ export function txToDbRows(tx: PartialJsonTransaction, height?: number) {
   const wallets = [] as { address: Buffer; public_modulus: Buffer }[];
 
   let contentType: string | undefined;
+  let contentEncoding: string | undefined;
+
   const txId = fromB64Url(tx.id);
 
   let transactionTagIndex = 0;
@@ -196,6 +202,10 @@ export function txToDbRows(tx: PartialJsonTransaction, height?: number) {
       contentType = tagValue.toString('utf8');
     }
 
+    if (isContentEncodingTag(tagName)) {
+      contentEncoding = tagValue.toString('utf8');
+    }
+
     newTxTags.push({
       tag_name_hash: tagNameHash,
       tag_value_hash: tagValueHash,
@@ -229,6 +239,7 @@ export function txToDbRows(tx: PartialJsonTransaction, height?: number) {
       data_size: tx.data_size,
       data_root: fromB64Url(tx.data_root),
       content_type: contentType,
+      content_encoding: contentEncoding,
       tag_count: tx.tags.length,
       indexed_at: currentUnixTimestamp(),
       height: height,
@@ -250,6 +261,8 @@ export function dataItemToDbRows(item: NormalizedDataItem, height?: number) {
   const wallets = [] as { address: Buffer; public_modulus: Buffer }[];
 
   let contentType: string | undefined;
+  let contentEncoding: string | undefined;
+
   const id = fromB64Url(item.id);
 
   let dataItemTagIndex = 0;
@@ -266,6 +279,10 @@ export function dataItemToDbRows(item: NormalizedDataItem, height?: number) {
       contentType = tagValue.toString('utf8');
     }
 
+    if (isContentEncodingTag(tagName)) {
+      contentEncoding = tagValue.toString('utf8');
+    }
+
     newDataItemTags.push({
       tag_name_hash: tagNameHash,
       tag_value_hash: tagValueHash,
@@ -308,6 +325,7 @@ export function dataItemToDbRows(item: NormalizedDataItem, height?: number) {
     bundleDataItem,
     newDataItem: {
       anchor: fromB64Url(item.anchor),
+      content_encoding: contentEncoding ?? item.content_encoding,
       content_type: contentType ?? item.content_type,
       data_offset: item.data_offset,
       data_size: item.data_size,
@@ -937,6 +955,7 @@ export class StandaloneSqliteDatabaseWorker {
       dataRoot: dataRoot ? toB64Url(dataRoot) : undefined,
       size: coreRow?.data_size ?? dataRow?.data_size,
       contentType,
+      contentEncoding: coreRow?.content_encoding,
       isManifest: contentType === MANIFEST_CONTENT_TYPE,
       stable: coreRow?.stable === true,
       verified: dataRow?.verified === true,

src/lib/ans-104.ts

@@ -77,6 +77,7 @@ export function normalizeAns104DataItem({
   dataHash: string;
 }): NormalizedDataItem {
   let contentType: string | undefined;
+  let contentEncoding: string | undefined;
 
   const tags = ans104DataItem.tags.map(
     (tag: { name: string; value: string }) => {
@@ -88,6 +89,14 @@ export function normalizeAns104DataItem({
         contentType = tag.value;
       }
 
+      // get content encoding from the first content-type tag
+      if (
+        contentEncoding === undefined &&
+        tag.name.toLowerCase() === 'content-encoding'
+      ) {
+        contentEncoding = tag.value;
+      }
+
       return {
         name: utf8ToB64Url(tag.name),
         value: utf8ToB64Url(tag.value),
@@ -97,6 +106,7 @@ export function normalizeAns104DataItem({
 
   return {
     anchor: ans104DataItem.anchor,
+    content_encoding: contentEncoding,
     content_type: contentType,
     data_hash: dataHash,
     data_offset: ans104DataItem.dataOffset,

src/routes/data/handlers.ts

@@ -95,6 +95,10 @@ const setDataHeaders = ({
       data.sourceContentType ??
       DEFAULT_CONTENT_TYPE,
   );
+
+  if (dataAttributes?.contentEncoding !== undefined) {
+    res.header('Content-Encoding', dataAttributes.contentEncoding);
+  }
 };
 
 const getRequestAttributes = (req: Request): RequestAttributes => {

src/types.d.ts

@@ -251,6 +251,7 @@ export interface NestedDataIndexWriter {
 
 export interface NormalizedBundleDataItem {
   anchor: string;
+  content_encoding?: string;
   content_type?: string;
   data_hash: string;
   data_offset: number;
@@ -277,6 +278,7 @@ export interface NormalizedBundleDataItem {
 
 export interface NormalizedOptimisticDataItem {
   anchor: string;
+  content_encoding?: string;
   content_type?: string;
   data_hash: null;
   data_offset: null;
@@ -495,6 +497,7 @@ export interface ContiguousDataAttributes {
   hash?: string;
   dataRoot?: string;
   size: number;
+  contentEncoding?: string;
   contentType: string | undefined;
   isManifest: boolean;
   stable: boolean;

test/bundles-schema.sql

@@ -23,7 +23,7 @@ CREATE TABLE stable_data_items (
 
   -- Metadata
   tag_count INTEGER NOT NULL,
-  indexed_at INTEGER NOT NULL, signature_type INTEGER, offset INTEGER, size INTEGER, owner_offset INTEGER, owner_size INTEGER, signature_offset INTEGER, signature_size INTEGER,
+  indexed_at INTEGER NOT NULL, signature_type INTEGER, offset INTEGER, size INTEGER, owner_offset INTEGER, owner_size INTEGER, signature_offset INTEGER, signature_size INTEGER, content_encoding TEXT,
   PRIMARY KEY (id)
 );
 CREATE INDEX stable_data_items_height_block_transaction_index_id_idx ON stable_data_items (height, block_transaction_index, id);
@@ -119,7 +119,7 @@ CREATE TABLE new_data_items (
 
   -- Metadata
   tag_count INTEGER NOT NULL,
-  indexed_at INTEGER NOT NULL, signature_type INTEGER, offset INTEGER, size INTEGER, owner_offset INTEGER, owner_size INTEGER, signature_offset INTEGER, signature_size INTEGER,
+  indexed_at INTEGER NOT NULL, signature_type INTEGER, offset INTEGER, size INTEGER, owner_offset INTEGER, owner_size INTEGER, signature_offset INTEGER, signature_size INTEGER, content_encoding TEXT,
   PRIMARY KEY (id)
 );
 CREATE INDEX new_data_items_parent_id_id_idx ON new_data_items (parent_id, id);

test/core-schema.sql

@@ -36,7 +36,7 @@ CREATE TABLE stable_transactions (
 
   -- Metadata
   tag_count INTEGER NOT NULL
-, offset INTEGER);
+, offset INTEGER, content_encoding TEXT);
 CREATE INDEX stable_transactions_id_height_block_transaction_index_idx ON stable_transactions (height, block_transaction_index);
 CREATE INDEX stable_transactions_target_height_block_transaction_index_idx ON stable_transactions (target, height, block_transaction_index);
 CREATE INDEX stable_transactions_owner_address_height_block_transaction_index_idx ON stable_transactions (owner_address, height, block_transaction_index);
@@ -130,7 +130,7 @@ CREATE TABLE new_transactions (
   -- Metadata
   tag_count INTEGER NOT NULL,
   indexed_at INTEGER NOT NULL
-, height INTEGER);
+, height INTEGER, content_encoding TEXT);
 CREATE INDEX new_transactions_target_id_idx ON new_transactions (target, id);
 CREATE INDEX new_transactions_owner_address_id_idx ON new_transactions (owner_address, id);
 CREATE TABLE new_block_transactions (

test/end-to-end/indexing.test.ts

@@ -27,7 +27,7 @@ import axios from 'axios';
 import { default as wait } from 'wait';
 import Sqlite, { Database } from 'better-sqlite3';
 import crypto from 'node:crypto';
-import { b64UrlToUtf8, toB64Url } from '../../src/lib/encoding.js';
+import { b64UrlToUtf8, toB64Url, fromB64Url } from '../../src/lib/encoding.js';
 import { getMaxHeight, waitForBlocks } from './utils.js';
 import { Environment } from 'testcontainers/build/types.js';
 
@@ -631,4 +631,95 @@ describe('Indexing', function () {
       });
     });
   });
+
+  describe('Content-Encoding', function () {
+    const txId = 'NT9b6xQqxMGNsbp1h6N-pmd-YM0hWPP3KDcM2EA1Hk8';
+    const bundleId = '0WUql4Qv3OFf-e9PR2hnZM1wv9s5TPbub7uvZXaQf5w';
+    const dataItemId = 'XC6f7QFAxkSHltkW96fDz-hwUU_ntRS-cpiT2wTe8oA';
+    let bundlesDb: Database;
+    let coreDb: Database;
+    let compose: StartedDockerComposeEnvironment;
+
+    const waitForIndexing = async () => {
+      const getAllTxs = () =>
+        coreDb.prepare('SELECT * FROM new_transactions').all();
+
+      const getAllDI = () =>
+        bundlesDb.prepare('SELECT * FROM new_data_items').all();
+
+      while (getAllTxs().length === 0 || getAllDI().length === 0) {
+        console.log('Waiting for pending txs and data items to be indexed...');
+        await wait(5000);
+      }
+    };
+
+    before(async function () {
+      compose = await composeUp({
+        ARNS_ROOT_HOST: '',
+      });
+
+      await axios({
+        method: 'post',
+        url: 'http://localhost:4000/ar-io/admin/queue-tx',
+        headers: {
+          Authorization: 'Bearer secret',
+          'Content-Type': 'application/json',
+        },
+        data: { id: txId },
+      });
+
+      await axios({
+        method: 'post',
+        url: 'http://localhost:4000/ar-io/admin/queue-bundle',
+        headers: {
+          Authorization: 'Bearer secret',
+          'Content-Type': 'application/json',
+        },
+        data: { id: bundleId },
+      });
+
+      bundlesDb = new Sqlite(`${projectRootPath}/data/sqlite/bundles.db`);
+      coreDb = new Sqlite(`${projectRootPath}/data/sqlite/core.db`);
+
+      await waitForIndexing();
+    });
+
+    after(async function () {
+      await compose.down();
+    });
+
+    it('Verifying if transaction content-encoding was indexed', async function () {
+      const stmt = coreDb.prepare(
+        'SELECT * FROM new_transactions WHERE id = @id',
+      );
+      const transaction = stmt.get({ id: fromB64Url(txId) });
+
+      assert.equal(transaction.content_encoding, 'gzip');
+    });
+
+    it('Verifying if content-encoding header is sent', async function () {
+      const res = await axios.head(`http://localhost:4000/raw/${txId}`, {
+        decompress: false,
+      });
+
+      assert.equal(res.headers['content-encoding'], 'gzip');
+    });
+
+    it('Verifying if data item content-encoding was indexed', async function () {
+      const stmt = bundlesDb.prepare(
+        'SELECT * FROM new_data_items WHERE id = @id',
+      );
+      const dataItem = stmt.get({ id: fromB64Url(dataItemId) });
+
+      assert.equal(dataItem.content_encoding, 'gzip');
+    });
+
+    it('Verifying if content-encoding header is sent', async function () {
+      const res = await axios.head(`http://localhost:4000/raw/${dataItemId}`, {
+        decompress: false,
+      });
+
+      assert.equal(res.headers['content-encoding'], 'gzip');
+    });
+  });
 });