feat: add options to disable abort/timeout mechanisms

CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [Unreleased]
+
+### Added
+
+### Changed
+
+### Fixed
+
 ## [Release 72] - 2026-03-11
 
 This is a **recommended release** focused on **data retrieval reliability** and **caching intelligence**. Key highlights include a **negative data cache** that reduces upstream load for consistently missing data, **direct byte offset hints** to help gateways locate data when internal lookup mechanisms fall short, **untrusted data caching with stochastic re-verification**, and significant **stream reliability improvements** that eliminate false timeouts on large transfers. It also adds **gateway loop prevention** via per-gateway via-chain detection.

docker-compose.yaml

@@ -1,7 +1,7 @@
 ---
 services:
   envoy:
-    image: ghcr.io/ar-io/ar-io-envoy:${ENVOY_IMAGE_TAG:-17a2cbdb71e1d1eba1a3c4e29aff96d69feb3246}
+    image: ghcr.io/ar-io/ar-io-envoy:${ENVOY_IMAGE_TAG:-latest}
     build:
       context: envoy/
     restart: unless-stopped
@@ -41,7 +41,7 @@ services:
       - observer
 
   core:
-    image: ghcr.io/ar-io/ar-io-core:${CORE_IMAGE_TAG:-fb4017499c42a60d81bf5d0624a26b84841cd005}
+    image: ghcr.io/ar-io/ar-io-core:${CORE_IMAGE_TAG:-latest}
     build:
       context: .
     restart: unless-stopped
@@ -69,6 +69,9 @@ services:
       - TRUSTED_GATEWAY_URL=${TRUSTED_GATEWAY_URL:-}
       - TRUSTED_GATEWAYS_URLS=${TRUSTED_GATEWAYS_URLS:-}
       - TRUSTED_GATEWAYS_REQUEST_TIMEOUT_MS=${TRUSTED_GATEWAYS_REQUEST_TIMEOUT_MS:-}
+      - TRUSTED_GATEWAYS_DISABLE_REQUEST_TIMEOUT_ABORTS=${TRUSTED_GATEWAYS_DISABLE_REQUEST_TIMEOUT_ABORTS:-}
+      - TRUSTED_GATEWAYS_DISABLE_STREAM_STALL_ABORTS=${TRUSTED_GATEWAYS_DISABLE_STREAM_STALL_ABORTS:-}
+      - DISABLE_REQUEST_ABORT_SIGNAL=${DISABLE_REQUEST_ABORT_SIGNAL:-}
       - TRUSTED_GATEWAYS_BLOCKED_ORIGINS=${TRUSTED_GATEWAYS_BLOCKED_ORIGINS:-}
       - TRUSTED_GATEWAYS_BLOCKED_IPS_AND_CIDRS=${TRUSTED_GATEWAYS_BLOCKED_IPS_AND_CIDRS:-}
       - BUNDLER_URLS=${BUNDLER_URLS:-}
@@ -185,7 +188,7 @@ services:
       - AWS_ELASTICACHE_TURBO_HOST=${AWS_ELASTICACHE_TURBO_HOST:-}
       - AWS_ELASTICACHE_TURBO_USE_TLS=${AWS_ELASTICACHE_TURBO_USE_TLS:-}
       - AWS_ELASTICACHE_TURBO_PORT=${AWS_ELASTICACHE_TURBO_PORT:-}
-      - AR_IO_NODE_RELEASE=${AR_IO_NODE_RELEASE:-72}
+      - AR_IO_NODE_RELEASE=${AR_IO_NODE_RELEASE:-73-pre}
       - CHUNK_POST_MIN_SUCCESS_COUNT=${CHUNK_POST_MIN_SUCCESS_COUNT:-}
       - CHUNK_POST_SORTED_PEERS_CACHE_DURATION_MS=${CHUNK_POST_SORTED_PEERS_CACHE_DURATION_MS:-}
       - CHUNK_POST_RESPONSE_TIMEOUT_MS=${CHUNK_POST_RESPONSE_TIMEOUT_MS:-}
@@ -367,7 +370,7 @@ services:
       - ar-io-network
 
   clickhouse-auto-import:
-    image: ghcr.io/ar-io/ar-io-clickhouse-auto-import:${CLICKHOUSE_AUTO_IMPORT_IMAGE_TAG:-4512361f3d6bdc0d8a44dd83eb796fd88804a384}
+    image: ghcr.io/ar-io/ar-io-clickhouse-auto-import:${CLICKHOUSE_AUTO_IMPORT_IMAGE_TAG:-latest}
     profiles:
       - clickhouse
     build:
@@ -421,7 +424,7 @@ services:
       - TURBO_UPLOAD_SERVICE_URL=${TURBO_UPLOAD_SERVICE_URL:-}
       - RUN_OBSERVER=${RUN_OBSERVER:-true}
       - MIN_RELEASE_NUMBER=${MIN_RELEASE_NUMBER:-0}
-      - AR_IO_NODE_RELEASE=${AR_IO_NODE_RELEASE:-72}
+      - AR_IO_NODE_RELEASE=${AR_IO_NODE_RELEASE:-73-pre}
       - AR_IO_SDK_LOG_LEVEL=${AR_IO_SDK_LOG_LEVEL:-none}
       - AO_CU_URL=${AO_CU_URL:-}
       - NETWORK_AO_CU_URL=${NETWORK_AO_CU_URL:-}
@@ -453,7 +456,7 @@ services:
       - ar-io-network
 
   litestream:
-    image: ghcr.io/ar-io/ar-io-litestream:${LITESTREAM_IMAGE_TAG:-be121fc0ae24a9eb7cdb2b92d01f047039b5f5e8}
+    image: ghcr.io/ar-io/ar-io-litestream:${LITESTREAM_IMAGE_TAG:-latest}
     build:
       context: litestream/
       dockerfile: Dockerfile

docs/envs.md

@@ -12,6 +12,9 @@ This document describes the environment variables that can be used to configure
 | TRUSTED_GATEWAY_URL                              | String               | "https://arweave.net"                         | Arweave node to use for proxying requests                                                                                                                            |
 | TRUSTED_GATEWAYS_URLS                            | String               | See below                                     | A JSON map of gateway URLs to priority (number) or config object (`{"priority": N, "trusted": bool}`). Simple number values are implicitly trusted. Default: `{"https://turbo-gateway.com": 1, "https://arweave.net": {"priority": 2, "trusted": false}}`. When `trusted` is false, data is only cached if a hash is already known and matches; the hash is never written to the DB as authoritative. |
 | TRUSTED_GATEWAYS_REQUEST_TIMEOUT_MS              | String               | "10000"                                       | Connection timeout in milliseconds for trusted gateways (time to receive response headers)                                                                           |
+| TRUSTED_GATEWAYS_DISABLE_REQUEST_TIMEOUT_ABORTS  | Boolean              | false                                         | If true, disables the AbortController-based connection timeout for trusted gateway requests. Client disconnect aborts and stream stall timeouts still apply.        |
+| TRUSTED_GATEWAYS_DISABLE_STREAM_STALL_ABORTS     | Boolean              | false                                         | If true, disables stream stall aborts for trusted gateway responses. Connection timeout and client disconnect aborts still apply unless separately disabled.        |
+| DISABLE_REQUEST_ABORT_SIGNAL                     | Boolean              | false                                         | If true, the request abort signal middleware does not abort `req.signal` when the client disconnects. This disables client-disconnect abort propagation globally for request-scoped operations. |
 | STREAM_STALL_TIMEOUT_MS                          | String               | "30000"                                       | Stall timeout in milliseconds for data streams from gateways and peers. Stream is aborted if no data is received for this duration. Prevents stalled transfers from hanging indefinitely while allowing large, actively-streaming transfers to complete. |
 | TRUSTED_GATEWAYS_BLOCKED_ORIGINS                 | String               | ""                                            | Comma-separated list of X-AR-IO-Origin header values to block when forwarding to trusted gateways (prevents loops and blocks unwanted sources)                   |
 | TRUSTED_GATEWAYS_BLOCKED_IPS_AND_CIDRS           | String               | ""                                            | Comma-separated list of IPs and CIDR ranges to block when forwarding to trusted gateways (prevents forwarding requests from specific client IPs)                |

src/app.ts

@@ -71,7 +71,11 @@ app.use(
 );
 
 // Attach AbortSignal to all requests for client disconnect handling
-app.use(createAbortSignalMiddleware());
+app.use(
+  createAbortSignalMiddleware({
+    disableRequestAbortSignal: config.DISABLE_REQUEST_ABORT_SIGNAL,
+  }),
+);
 
 app.use(
   createX402Router({

src/config.ts

@@ -198,6 +198,19 @@ export const TRUSTED_GATEWAYS_REQUEST_TIMEOUT_MS = +env.varOrDefault(
   '10000',
 );
 
+export const TRUSTED_GATEWAYS_DISABLE_REQUEST_TIMEOUT_ABORTS =
+  env.varOrDefault(
+    'TRUSTED_GATEWAYS_DISABLE_REQUEST_TIMEOUT_ABORTS',
+    'false',
+  ) === 'true';
+
+export const TRUSTED_GATEWAYS_DISABLE_STREAM_STALL_ABORTS =
+  env.varOrDefault('TRUSTED_GATEWAYS_DISABLE_STREAM_STALL_ABORTS', 'false') ===
+  'true';
+
+export const DISABLE_REQUEST_ABORT_SIGNAL =
+  env.varOrDefault('DISABLE_REQUEST_ABORT_SIGNAL', 'false') === 'true';
+
 export const STREAM_STALL_TIMEOUT_MS = env.positiveIntOrDefault(
   'STREAM_STALL_TIMEOUT_MS',
   1000 * 30, // 30 seconds

src/data/gateways-data-source.test.ts

@@ -647,6 +647,77 @@ describe('GatewayDataSource', () => {
         // Both gateways should be tried on timeout
         assert.equal(requestLog.length, 2);
       });
+
+      it('should disable trusted gateway timeout aborts when configured', async () => {
+        const dataSourceNoTimeoutAbort = new GatewaysDataSource({
+          log,
+          trustedGatewaysUrls: {
+            'https://gateway1.com': { priority: 1, trusted: true },
+          },
+          disableRequestTimeoutAborts: true,
+          requestTimeoutMs: 1,
+        });
+
+        let receivedSignal: AbortSignal | undefined;
+
+        mock.method(axios, 'create', (config: any) => ({
+          request: async (params: any) => {
+            receivedSignal = params.signal;
+            await new Promise((resolve) => setTimeout(resolve, 10));
+            return {
+              status: 200,
+              headers: {
+                'content-length': '123',
+                'content-type': 'text/plain',
+              },
+              data: axiosStreamData,
+            };
+          },
+          ...axiosMockCommonParams(config),
+        }));
+
+        const data = await dataSourceNoTimeoutAbort.getData({ id: 'test-id' });
+
+        assert.equal(data.size, 123);
+        assert.equal(receivedSignal?.aborted, false);
+      });
+
+      it('should disable trusted gateway stream stall aborts when configured', async () => {
+        const dataSourceNoStallAbort = new GatewaysDataSource({
+          log,
+          trustedGatewaysUrls: {
+            'https://gateway1.com': { priority: 1, trusted: true },
+          },
+          disableStreamStallAborts: true,
+          streamStallTimeoutMs: 1,
+        });
+
+        mock.method(axios, 'create', (config: any) => ({
+          request: async () => ({
+            status: 200,
+            headers: {
+              'content-length': '4',
+              'content-type': 'text/plain',
+            },
+            data: Readable.from(
+              (async function* () {
+                await new Promise((resolve) => setTimeout(resolve, 10));
+                yield 'test';
+              })(),
+            ),
+          }),
+          ...axiosMockCommonParams(config),
+        }));
+
+        const data = await dataSourceNoStallAbort.getData({ id: 'test-id' });
+        let received = '';
+        for await (const chunk of data.stream) {
+          received += chunk;
+        }
+
+        assert.equal(data.size, 4);
+        assert.equal(received, 'test');
+      });
     });
 
     describe('fallbackToBasePath', () => {

src/data/gateways-data-source.ts

@@ -38,6 +38,8 @@ export class GatewaysDataSource implements ContiguousDataSource {
   private trustedGateways: Map<number, string[]>;
   private gatewayTrust: Map<string, boolean>;
   private readonly requestTimeoutMs: number;
+  private readonly disableRequestTimeoutAborts: boolean;
+  private readonly disableStreamStallAborts: boolean;
   private readonly streamStallTimeoutMs: number;
   private readonly fallbackToBasePath: boolean;
   private readonly maxHopsAllowed: number;
@@ -46,19 +48,25 @@ export class GatewaysDataSource implements ContiguousDataSource {
     log,
     trustedGatewaysUrls,
     requestTimeoutMs = config.TRUSTED_GATEWAYS_REQUEST_TIMEOUT_MS,
+    disableRequestTimeoutAborts = config.TRUSTED_GATEWAYS_DISABLE_REQUEST_TIMEOUT_ABORTS,
+    disableStreamStallAborts = config.TRUSTED_GATEWAYS_DISABLE_STREAM_STALL_ABORTS,
     streamStallTimeoutMs = config.STREAM_STALL_TIMEOUT_MS,
     fallbackToBasePath = false,
     maxHopsAllowed = MAX_DATA_HOPS,
   }: {
     log: winston.Logger;
     trustedGatewaysUrls: Record<string, TrustedGatewayConfig>;
     requestTimeoutMs?: number;
+    disableRequestTimeoutAborts?: boolean;
+    disableStreamStallAborts?: boolean;
     streamStallTimeoutMs?: number;
     fallbackToBasePath?: boolean;
     maxHopsAllowed?: number;
   }) {
     this.log = log.child({ class: this.constructor.name });
     this.requestTimeoutMs = requestTimeoutMs;
+    this.disableRequestTimeoutAborts = disableRequestTimeoutAborts;
+    this.disableStreamStallAborts = disableStreamStallAborts;
     this.streamStallTimeoutMs = streamStallTimeoutMs;
     this.fallbackToBasePath = fallbackToBasePath;
     this.maxHopsAllowed = maxHopsAllowed;
@@ -105,6 +113,10 @@ export class GatewaysDataSource implements ContiguousDataSource {
           'arns.basename': requestAttributes?.arnsBasename,
           'gateways.config.priority_tiers': this.trustedGateways.size,
           'gateways.config.request_timeout_ms': this.requestTimeoutMs,
+          'gateways.config.disable_request_timeout_aborts':
+            this.disableRequestTimeoutAborts,
+          'gateways.config.disable_stream_stall_aborts':
+            this.disableStreamStallAborts,
         },
       },
       parentSpan,
@@ -218,10 +230,12 @@ export class GatewaysDataSource implements ContiguousDataSource {
               // for establishing the connection, then switch to stall-based
               // timeout once the stream starts flowing.
               const controller = new AbortController();
-              const connectionTimer = setTimeout(
-                () => controller.abort(new Error('Connection timeout')),
-                this.requestTimeoutMs,
-              );
+              const connectionTimer = this.disableRequestTimeoutAborts
+                ? undefined
+                : setTimeout(
+                    () => controller.abort(new Error('Connection timeout')),
+                    this.requestTimeoutMs,
+                  );
               const onClientAbort = () => controller.abort(signal?.reason);
               if (signal?.aborted) {
                 onClientAbort();
@@ -304,7 +318,9 @@ export class GatewaysDataSource implements ContiguousDataSource {
 
                 // Connection established - clear connection timeout and
                 // switch to stall-based timeout for the streaming phase
-                clearTimeout(connectionTimer);
+                if (connectionTimer !== undefined) {
+                  clearTimeout(connectionTimer);
+                }
                 if (signal) {
                   signal.removeEventListener('abort', onClientAbort);
                 }
@@ -319,7 +335,9 @@ export class GatewaysDataSource implements ContiguousDataSource {
                   );
                 }
 
-                attachStallTimeout(stream, this.streamStallTimeoutMs);
+                if (!this.disableStreamStallAborts) {
+                  attachStallTimeout(stream, this.streamStallTimeoutMs);
+                }
 
                 const gatewayTrusted =
                   this.gatewayTrust.get(gatewayUrl) ?? true;
@@ -394,7 +412,9 @@ export class GatewaysDataSource implements ContiguousDataSource {
                   }),
                 };
               } catch (rawError: any) {
-                clearTimeout(connectionTimer);
+                if (connectionTimer !== undefined) {
+                  clearTimeout(connectionTimer);
+                }
                 if (signal) {
                   signal.removeEventListener('abort', onClientAbort);
                 }

src/middleware/abort-signal.test.ts

@@ -0,0 +1,56 @@
+/**
+ * AR.IO Gateway
+ * Copyright (C) 2022-2025 Permanent Data Solutions, Inc. All Rights Reserved.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+import { strict as assert } from 'node:assert';
+import { describe, it } from 'node:test';
+import { EventEmitter } from 'node:events';
+import { createAbortSignalMiddleware } from './abort-signal.js';
+
+describe('createAbortSignalMiddleware', () => {
+  it('should abort req.signal when client disconnects before response completes', () => {
+    const middleware = createAbortSignalMiddleware();
+    const req = new EventEmitter() as any;
+    const res = { writableEnded: false } as any;
+    let nextCalled = false;
+
+    middleware(req, res, () => {
+      nextCalled = true;
+    });
+
+    assert.equal(nextCalled, true);
+    assert.equal(req.signal.aborted, false);
+
+    req.emit('close');
+
+    assert.equal(req.signal.aborted, true);
+  });
+
+  it('should not abort req.signal when disableRequestAbortSignal is true', () => {
+    const middleware = createAbortSignalMiddleware({
+      disableRequestAbortSignal: true,
+    });
+    const req = new EventEmitter() as any;
+    const res = { writableEnded: false } as any;
+
+    middleware(req, res, () => {});
+
+    req.emit('close');
+
+    assert.equal(req.signal.aborted, false);
+  });
+
+  it('should not abort req.signal after response has completed', () => {
+    const middleware = createAbortSignalMiddleware();
+    const req = new EventEmitter() as any;
+    const res = { writableEnded: true } as any;
+
+    middleware(req, res, () => {});
+
+    req.emit('close');
+
+    assert.equal(req.signal.aborted, false);
+  });
+});

src/middleware/abort-signal.ts

@@ -17,16 +17,22 @@ import { Handler, Request, Response } from 'express';
  * This allows downstream operations to be cancelled when clients disconnect,
  * preventing wasted work on requests that will never be delivered.
  */
-export function createAbortSignalMiddleware(): Handler {
+export function createAbortSignalMiddleware({
+  disableRequestAbortSignal = false,
+}: {
+  disableRequestAbortSignal?: boolean;
+} = {}): Handler {
   return (req: Request, res: Response, next) => {
     const controller = new AbortController();
 
-    // Abort when client disconnects before response completes
-    req.on('close', () => {
-      if (!res.writableEnded) {
-        controller.abort();
-      }
-    });
+    if (!disableRequestAbortSignal) {
+      // Abort when client disconnects before response completes
+      req.on('close', () => {
+        if (!res.writableEnded) {
+          controller.abort();
+        }
+      });
+    }
 
     // Attach signal to request for use by handlers
     req.signal = controller.signal;

src/version.ts

@@ -5,4 +5,4 @@
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
 
-export const release = '72';
+export const release = '73-pre';