Merge branch 'develop' of https://github.com/ar-io/ar-io-node into de…

…velop

.env.example

@@ -0,0 +1,56 @@
+# Starting block height for node synchronization (0 = start from the beginning)
+# START_HEIGHT=0
+
+# Stop block height for node synchronization (Infinity = keep syncing until stopped)
+# STOP_HEIGHT="Infinity"
+
+# Arweave node to use for fetching data
+# TRUSTED_NODE_URL="https://arweave.net"
+
+# Arweave node to use for proxying requests
+# TRUSTED_GATEWAY_URL="https://arweave.net"
+
+# ArNS gateway
+# TRUSTED_ARNS_GATEWAY_URL="https://__NAME__.arweave.dev"
+
+# If true, skips the local cache and always fetches headers from the node
+# SKIP_CACHE=false
+
+# Adds an "INSTANCE_ID" field to output logs
+# INSTANCE_ID=""
+
+# Sets the format of output logs, accepts "simple" and "json"
+# LOG_FORMAT="simple"
+
+# AR.IO node exposed port number
+# PORT=4000
+
+# Number from 0 to 1, representing the probability of a request failing
+# SIMULATED_REQUEST_FAILURE_RATE=0
+
+# Arweave wallet address used for staking and rewards
+# AR_IO_WALLET=""
+
+# Admin key used for accessing the admin API
+# ADMIN_API_KEY="secret"
+
+# If true, ar.io node will start indexing missing bundles
+# BACKFILL_BUNDLE_RECORDS=false
+
+# If true, all indexed bundles will be reprocessed with the new filters (you can use this when you change the filters)
+# FILTER_CHANGE_REPROCESS=false
+
+# Only bundles compliant with this filter will be unbundled
+# ANS104_UNBUNDLE_FILTER={"never": true}
+
+# Only bundles compliant with this filter will be indexed
+# ANS104_INDEX_FILTER={"never": true}
+
+# Root host for ArNS
+# ARNS_ROOT_HOST=""
+
+# Protocol setting in process of creating sandbox domain in ArNS (ARNS_ROOT_HOST needs to be set for this env to have any effect)
+# SANDBOX_PROTOCOL=""
+
+# If true, start indexing blocks, tx, ANS104 bundles
+# START_WRITERS=true

.gitignore

@@ -18,5 +18,8 @@ test-results.json
 /.vscode
 .DS_Store
 
+# JetBrains
+/.idea
+
 # Exceptions
 !/data/.gitkeep

CONTRIBUTING.md

@@ -10,6 +10,8 @@ NOTE: Contribution implies licensing under the terms of [LICENSE]
 - Ensure commit messages adequately explain the reasons for the changes.
 - Ensure changes are consistent with the [principles and practices]
   outlined in the README.
+- Ensure that tests were added or that the manual testing process followed is
+  described in the PR.
 - Run [lint].
 - Run the [unit tests].
 
@@ -18,7 +20,7 @@ NOTE: Contribution implies licensing under the terms of [LICENSE]
 - Keep the first line short but informative.
 - Provide explanation of why the change is being made in the commit message
   body.
-- Prefer copying relevant information into the commit body over linking to them.
+- Prefer copying relevant information into the commit body over linking to it.
 - Consider whether your commit message includes enough detail for someone to be
   able to understand it in the future.
 

Dockerfile

@@ -26,7 +26,7 @@ VOLUME /app/data
 
 # EXPOSE PORT AND SETUP HEALTHCHECK
 EXPOSE 4000
-HEALTHCHECK CMD curl --fail http://localhost:4000/healthcheck || exit 1
+HEALTHCHECK CMD curl --fail http://localhost:4000/ar-io/healthcheck || exit 1
 
 # ADD LABELS
 LABEL org.opencontainers.image.title="ar.io Core Service"

docs/envs.md

@@ -0,0 +1,24 @@
+# ENVs
+This document describes the environment variables that can be used to configure the `ar.io` node.
+
+| ENV_NAME                       | TYPE                 | DEFAULT_VALUE                  | DESCRIPTION                                                                                                                     |
+|--------------------------------|----------------------|--------------------------------|---------------------------------------------------------------------------------------------------------------------------------|
+| START_HEIGHT                   | Number or "Infinity" | 0                              | Starting block height for node synchronization (0 = start from the beginning)                                                   |
+| STOP_HEIGHT                    | Number or "Infinity" | "Infinity"                     | Stop block height for node synchronization (Infinity = keep syncing until stopped)                                              |
+| TRUSTED_NODE_URL               | String               | "https://arweave.net"          | Arweave node to use for fetching data                                                                                           |
+| TRUSTED_GATEWAY_URL            | String               | "https://arweave.net"          | Arweave node to use for proxying requests                                                                                       |
+| TRUSTED_ARNS_GATEWAY_URL       | String               | "https://__NAME__.arweave.dev" | ArNS gateway                                                                                                                    |
+| INSTANCE_ID                    | String               | ""                             | Adds an "INSTACE_ID" field to output logs                                                                                       |
+| LOG_FORMAT                     | String               | "simple"                       | Sets the format of output logs, accepts "simple" and "json"                                                                     |
+| SKIP_CACHE                     | Boolean              | false                          | If true, skips the local cache and always fetches headers from the node                                                         |
+| PORT                           | Number               | 4000                           | AR.IO node exposed port number                                                                                                  |
+| SIMULATED_REQUEST_FAILURE_RATE | Number               | 0                              | Number from 0 to 1, representing the probability of a request failing                                                           |
+| AR_IO_WALLET                   | String               | ""                             | Arweave wallet address used for staking and rewards                                                                             |
+| ADMIN_API_KEY                  | String               | Generated                      | API key used for admin API requests (if not set, it's generated and logged into the console)                                    |
+| BACKFILL_BUNDLE_RECORDS        | Boolean              | false                          | If true, ar.io node will start indexing missing bundles                                                                         |
+| FILTER_CHANGE_REPROCESS        | Boolean              | false                          | If true, all indexed bundles will be reprocessed with the new filters (you can use this when you change the filters)            |
+| ANS104_UNBUNDLE_FILTER         | String               | '{"never": true}'              | Only bundles compliant with this filter will be unbundled                                                                       |
+| ANS104_INDEX_FILTER            | String               | '{"never": true}'              | Only bundles compliant with this filter will be indexed                                                                         |
+| ARNS_ROOT_HOST                 | String               | undefined                      | Domain name for ArNS host                                                                                                       |
+| SANDBOX_PROTOCOL               | String               | undefined                      | Protocol setting in process of creating sandbox domain in ArNS (ARNS_ROOT_HOST needs to be set for this env to have any effect) |
+| START_WRITERS                  | Boolean              | true                           | If true, start indexing blocks, tx, ANS104 bundles                                                                              |

envoy/envoy.template.yaml

@@ -40,13 +40,6 @@ static_resources:
                               retry_on: '5xx,reset,retriable-status-codes'
                               retriable_status_codes: 404
                               num_retries: 5
-                        - match: { prefix: '/healthcheck' }
-                          route:
-                            cluster: ario_gateways
-                            retry_policy:
-                              retry_on: '5xx,reset,retriable-status-codes'
-                              retriable_status_codes: 404
-                              num_retries: 5
                         - match: { prefix: '/graphql' }
                           route:
                             cluster: graphql_gateways

src/app.ts

@@ -89,7 +89,6 @@ if (config.ARNS_ROOT_HOST !== undefined) {
 
   app.use(
     createSandboxMiddleware({
-      rootHost: config.ARNS_ROOT_HOST,
       sandboxProtocol: config.SANDBOX_PROTOCOL,
     }),
   );

src/config.ts

@@ -70,6 +70,8 @@ export const ANS104_INDEX_FILTER = createFilter(
   JSON.parse(ANS104_INDEX_FILTER_STRING),
 );
 export const ARNS_ROOT_HOST = env.varOrUndefined('ARNS_ROOT_HOST');
+export const ROOT_HOST_SUBDOMAIN_LENGTH =
+  ARNS_ROOT_HOST !== undefined ? ARNS_ROOT_HOST.split('.').length - 2 : 0;
 export const SANDBOX_PROTOCOL = env.varOrUndefined('SANDBOX_PROTOCOL');
 export const START_WRITERS =
   env.varOrDefault('START_WRITERS', 'true') === 'true';

src/middleware/arns.ts

@@ -18,6 +18,7 @@
 import { Handler } from 'express';
 import { asyncMiddleware } from 'middleware-async';
 
+import * as config from '../config.js';
 import { sendNotFound } from '../routes/data.js';
 import { NameResolver } from '../types.js';
 
@@ -32,26 +33,32 @@ export const createArnsMiddleware = ({
 }): Handler =>
   asyncMiddleware(async (req, res, next) => {
     if (
-      Array.isArray(req.subdomains) &&
-      req.subdomains.length === 1 &&
-      !EXCLUDED_SUBDOMAINS.has(req.subdomains[0]) &&
+      // Ignore subdomains that are part of the ArNS root hostname.
+      !Array.isArray(req.subdomains) ||
+      req.subdomains.length === config.ROOT_HOST_SUBDOMAIN_LENGTH
+    ) {
+      next();
+      return;
+    }
+    const arnsSubdomain = req.subdomains[req.subdomains.length - 1];
+    if (
+      EXCLUDED_SUBDOMAINS.has(arnsSubdomain) ||
       // Avoid collisions with sandbox URLs by ensuring the subdomain length
       // is below the mininimum length of a sandbox subdomain. Undernames are
       // are an exception because they can be longer and '_' cannot appear in
       // base32.
-      (req.subdomains[0].length <= 48 || req.subdomains[0].match(/_/))
+      (arnsSubdomain.length > 48 && !arnsSubdomain.match(/_/))
     ) {
-      const { resolvedId, ttl } = await nameResolver.resolve(req.subdomains[0]);
-      if (resolvedId !== undefined) {
-        res.header('X-ArNS-Resolved-Id', resolvedId);
-        res.header('X-ArNS-TTL-Seconds', ttl.toString());
-        res.header('Cache-Control', `public, max-age=${ttl}`);
-        dataHandler(req, res, next);
-        return;
-      } else {
-        sendNotFound(res);
-        return;
-      }
+      next();
+      return;
+    }
+    const { resolvedId, ttl } = await nameResolver.resolve(arnsSubdomain);
+    if (resolvedId === undefined) {
+      sendNotFound(res);
+      return;
     }
-    next();
+    res.header('X-ArNS-Resolved-Id', resolvedId);
+    res.header('X-ArNS-TTL-Seconds', ttl.toString());
+    res.header('Cache-Control', `public, max-age=${ttl}`);
+    dataHandler(req, res, next);
   });

src/middleware/sandbox.ts

@@ -19,11 +19,12 @@ import { Handler, Request } from 'express';
 import url from 'node:url';
 import { base32 } from 'rfc4648';
 
+import * as config from '../config.js';
 import { fromB64Url } from '../lib/encoding.js';
 
 function getRequestSandbox(req: Request): string | undefined {
-  if (req.subdomains.length === 1) {
-    return req.subdomains[0];
+  if (req.subdomains.length > config.ROOT_HOST_SUBDOMAIN_LENGTH) {
+    return req.subdomains[req.subdomains.length - 1];
   }
   return undefined;
 }
@@ -37,14 +38,12 @@ function sandboxFromId(id: string): string {
 }
 
 export function createSandboxMiddleware({
-  rootHost,
   sandboxProtocol,
 }: {
-  rootHost?: string;
   sandboxProtocol?: string;
 }): Handler {
   return (req, res, next) => {
-    if (rootHost === undefined) {
+    if (config.ARNS_ROOT_HOST === undefined) {
       next();
       return;
     }
@@ -63,7 +62,7 @@ export function createSandboxMiddleware({
       const protocol = sandboxProtocol ?? (req.secure ? 'https' : 'http');
       return res.redirect(
         302,
-        `${protocol}://${idSandbox}.${rootHost}${path}?${queryString}`,
+        `${protocol}://${idSandbox}.${config.ARNS_ROOT_HOST}${path}?${queryString}`,
       );
     }