From 586a2f9846a4a2a33605444ffb80b0bbef45737b Mon Sep 17 00:00:00 2001 From: VladimirPress Date: Wed, 19 Nov 2025 17:34:58 +0200 Subject: [PATCH 1/2] Update indexsource.py Adds a new per-collection configuration option use_indexer_date_filter. When enabled for a collection, pywb forwards from and to date parameters to the indexer, if present. This shifts date filtering to the indexer, reducing pywb CPU and memory usage, preventing OOM conditions, and lowering network traffic. --- pywb/warcserver/index/indexsource.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pywb/warcserver/index/indexsource.py b/pywb/warcserver/index/indexsource.py index 7572c56ac..3510ff69f 100644 --- a/pywb/warcserver/index/indexsource.py +++ b/pywb/warcserver/index/indexsource.py @@ -119,11 +119,12 @@ def init_from_config(cls, config): class RemoteIndexSource(BaseIndexSource): CDX_MATCH_RX = re.compile('^cdxj?\+(?Phttps?\:.*)') - def __init__(self, api_url, replay_url, url_field='load_url', closest_limit=100): + def __init__(self, api_url, replay_url, url_field='load_url', closest_limit=100, use_indexer_date_filter=False): self.api_url = api_url self.replay_url = replay_url self.url_field = url_field self.closest_limit = closest_limit + self.use_indexer_date_filter = use_indexer_date_filter self._init_sesh() def _get_api_url(self, params): @@ -133,7 +134,13 @@ def _get_api_url(self, params): if 'matchType' in params: api_url += '&matchType=' + params.get('matchType') - + + if self.use_indexer_date_filter: + if 'from' in params: + api_url += '&from=' + params.get('from') + if 'to' in params: + api_url += '&to=' + params.get('to') + return api_url def load_index(self, params): @@ -211,7 +218,9 @@ def init_from_config(cls, config): if config['type'] != 'cdx': return - return cls(config['api_url'], config['replay_url']) + use_indexer_date_filter = config.get('use_indexer_date_filter', False) + + return cls(config['api_url'], config['replay_url'], use_indexer_date_filter=use_indexer_date_filter) # ============================================================================= From acf980e7a8cfce837fa3434294904ed7fe329aa9 Mon Sep 17 00:00:00 2001 From: VladimirPress Date: Sun, 7 Dec 2025 12:59:27 +0200 Subject: [PATCH 2/2] Update indexsource.py Rename use_indexer_date_filter to use_remote_date_filter. --- pywb/warcserver/index/indexsource.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pywb/warcserver/index/indexsource.py b/pywb/warcserver/index/indexsource.py index 3510ff69f..45da9d327 100644 --- a/pywb/warcserver/index/indexsource.py +++ b/pywb/warcserver/index/indexsource.py @@ -119,12 +119,12 @@ def init_from_config(cls, config): class RemoteIndexSource(BaseIndexSource): CDX_MATCH_RX = re.compile('^cdxj?\+(?Phttps?\:.*)') - def __init__(self, api_url, replay_url, url_field='load_url', closest_limit=100, use_indexer_date_filter=False): + def __init__(self, api_url, replay_url, url_field='load_url', closest_limit=100, use_remote_date_filter=False): self.api_url = api_url self.replay_url = replay_url self.url_field = url_field self.closest_limit = closest_limit - self.use_indexer_date_filter = use_indexer_date_filter + self.use_remote_date_filter = use_remote_date_filter self._init_sesh() def _get_api_url(self, params): @@ -135,7 +135,7 @@ def _get_api_url(self, params): if 'matchType' in params: api_url += '&matchType=' + params.get('matchType') - if self.use_indexer_date_filter: + if self.use_remote_date_filter: if 'from' in params: api_url += '&from=' + params.get('from') if 'to' in params: @@ -218,9 +218,9 @@ def init_from_config(cls, config): if config['type'] != 'cdx': return - use_indexer_date_filter = config.get('use_indexer_date_filter', False) + use_remote_date_filter = config.get('use_remote_date_filter', False) - return cls(config['api_url'], config['replay_url'], use_indexer_date_filter=use_indexer_date_filter) + return cls(config['api_url'], config['replay_url'], use_remote_date_filter=use_remote_date_filter) # =============================================================================