From 15dcf8c0a26c99cfa3790013e90f781abecfe0d0 Mon Sep 17 00:00:00 2001 From: Jim Robinson <933148+jrobinso@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:26:10 -0500 Subject: [PATCH 1/3] Limit the wait for a slow search service Co-Authored-By: Claude Fable 5.2 --- js/browser.js | 3 +- js/feature/textFeatureSource.js | 4 ++- js/igv.d.ts | 1 + js/qtl/qtlTrack.js | 4 +-- js/searchFeatures.js | 31 +++++++++++++++++-- test/testSearch.js | 55 +++++++++++++++++++++++++++++++++ 6 files changed, 91 insertions(+), 7 deletions(-) diff --git a/js/browser.js b/js/browser.js index 7ece9ff2e..09a97d983 100755 --- a/js/browser.js +++ b/js/browser.js @@ -203,7 +203,8 @@ class Browser { endField: config.search.endField || "end", geneField: config.search.geneField || "gene", snpField: config.search.snpField || "snp", - resultsField: config.search.resultsField + resultsField: config.search.resultsField, + timeout: config.search.timeout } } } diff --git a/js/feature/textFeatureSource.js b/js/feature/textFeatureSource.js index 3590c5dad..ff6baeef6 100644 --- a/js/feature/textFeatureSource.js +++ b/js/feature/textFeatureSource.js @@ -274,7 +274,9 @@ class TextFeatureSource extends BaseFeatureSource { // Any chromosome will do, the whole file is read. chromosomeNames is optional, fall back to a loaded chromosome. const chr = this.genome.chromosomeNames?.[0] ?? this.genome.chromosomes.keys().next().value try { - await this.getFeatures({chr, start: 0, end: Number.MAX_SAFE_INTEGER}) + // One load, shared by searches that arrive while it is in progress + this.searchLoad ??= this.getFeatures({chr, start: 0, end: Number.MAX_SAFE_INTEGER}) + await this.searchLoad } catch (error) { console.error(error) this.featureMap = new Map() // Don't retry the load on every search diff --git a/js/igv.d.ts b/js/igv.d.ts index c3a12221c..68b030df9 100644 --- a/js/igv.d.ts +++ b/js/igv.d.ts @@ -508,6 +508,7 @@ interface CreateOptExtras { search?: { url: string; + timeout?: number; coords?: 0 | 1; chromosomeField?: string; startField?: string; diff --git a/js/qtl/qtlTrack.js b/js/qtl/qtlTrack.js index a4e2803fe..80e667984 100755 --- a/js/qtl/qtlTrack.js +++ b/js/qtl/qtlTrack.js @@ -291,8 +291,8 @@ class QTLTrack extends TrackBase { // possibly Expand region to bring phenotype in view const canonicalChrName = this.browser.genome.getChromosomeName(chr) - for (let term of genes) { - const feature = await searchFeatures(this.browser, term) + const features = await Promise.all([...genes].map(term => searchFeatures(this.browser, term))) + for (let feature of features) { if (feature) { if (canonicalChrName === this.browser.genome.getChromosomeName(feature.chr)) { start = Math.min(start, feature.start) diff --git a/js/searchFeatures.js b/js/searchFeatures.js index 2e668156f..da60122c0 100644 --- a/js/searchFeatures.js +++ b/js/searchFeatures.js @@ -6,8 +6,10 @@ import {igvxhr, StringUtils} from "../node_modules/igv-utils/src/index.js" * This module is separate to avoid circular dependencies between search.js and hgvs.js */ +// Applied to any search service whose config does not set a timeout, so a hung service cannot block a search +const DEFAULT_TIMEOUT = 5000 + const DEFAULT_SEARCH_CONFIG = { - timeout: 2000, type: "plain", url: 'https://igv.org/genomes/locus.php?genome=$GENOME$&name=$FEATURE$', coords: 0 @@ -17,6 +19,12 @@ const DEFAULT_SEARCH_CONFIG = { const MAX_NOT_FOUND = 100 const notFoundURLs = new Set() +// Services that timed out or could not be reached are skipped for a while, so a search of several terms, or a +// repeated search, waits for one timeout and not one per lookup. Keyed by the service url template; the value is +// the time (ms) at which the service will be tried again. +const SERVICE_RETRY_INTERVAL = 60000 +const unavailableServices = new Map() + /** * Search for a feature by name in MANE transcripts, searchable tracks, and web services * @param {Object} browser - The IGV browser instance @@ -73,8 +81,25 @@ async function searchWebService(browser, locus, searchConfig) { return undefined } - const options = searchConfig.timeout ? {timeout: searchConfig.timeout} : undefined - const result = await igvxhr.loadString(path, options) + const retryTime = unavailableServices.get(searchConfig.url) + if (retryTime !== undefined) { + if (Date.now() < retryTime) { + return undefined + } + unavailableServices.delete(searchConfig.url) + } + + let result + try { + result = await igvxhr.loadString(path, {timeout: searchConfig.timeout ?? DEFAULT_TIMEOUT}) + } catch (error) { + // Status 0 is a timeout or a service that could not be reached. That says nothing about this term, so + // the miss is not remembered, the service is rested instead. + if (error && error.status === 0) { + unavailableServices.set(searchConfig.url, Date.now() + SERVICE_RETRY_INTERVAL) + } + throw error + } const feature = await processSearchResult(browser, result, searchConfig) if (!feature) { diff --git a/test/testSearch.js b/test/testSearch.js index 27b8f10d7..4cab62c19 100644 --- a/test/testSearch.js +++ b/test/testSearch.js @@ -5,6 +5,7 @@ import {createGenome} from "./utils/MockGenome.js" import search, {parseLocusString, searchWebService} from "../js/search.js" import FeatureSource from "../js/feature/featureSource.js" import {searchFeatures} from "../js/searchFeatures.js" +import {igvxhr} from "../node_modules/igv-utils/src/index.js" const genome = createGenome() @@ -196,6 +197,60 @@ suite("testSearch", function () { assert.isUndefined(await searchWebService(browser, "nosuchgene", searchConfig)) }) + test("webservice that cannot be reached is rested", async function () { + + const searchConfig = {type: "plain", url: "https://unreachable.example/a?name=$FEATURE$", coords: 0} + const loadString = igvxhr.loadString + const requests = [] + igvxhr.loadString = async (url, options) => { + requests.push({url, options}) + const error = new Error("timeout") + error.status = 0 + throw error + } + try { + let failed = false + try { + await searchWebService(browser, "gene1", searchConfig) + } catch (e) { + failed = true + } + assert.isTrue(failed) + + // The service is not tried again for other terms, they are reported as not found + assert.isUndefined(await searchWebService(browser, "gene2", searchConfig)) + assert.isUndefined(await searchWebService(browser, "gene1", searchConfig)) + assert.equal(requests.length, 1) + + // A timeout is always set, the configured one if there is one + assert.isAbove(requests[0].options.timeout, 0) + await searchWebService(browser, "gene1", {...searchConfig, url: "https://unreachable.example/b?name=$FEATURE$", timeout: 123}).catch(() => {}) + assert.equal(requests[1].options.timeout, 123) + } finally { + igvxhr.loadString = loadString + } + }) + + test("webservice error for one term does not rest the service", async function () { + + const searchConfig = {type: "plain", url: "https://failing.example/?name=$FEATURE$", coords: 0} + const loadString = igvxhr.loadString + let requests = 0 + igvxhr.loadString = async () => { + requests++ + const error = new Error("bad request") + error.status = 400 + throw error + } + try { + await searchWebService(browser, "gene1", searchConfig).catch(() => {}) + await searchWebService(browser, "gene2", searchConfig).catch(() => {}) + assert.equal(requests, 2) + } finally { + igvxhr.loadString = loadString + } + }) + test("search non-indexed searchable track, genome without chromosomeNames", async function () { // Genome with no chromosome list, only the chromosome loaded for the initial locus From 2dd63b55539f687af8f1641d902da5771fb67220 Mon Sep 17 00:00:00 2001 From: Jim Robinson <933148+jrobinso@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:54:27 -0500 Subject: [PATCH 2/3] Rest a search service only on a real timeout Co-Authored-By: Claude Fable 5.2 --- js/feature/textFeatureSource.js | 2 ++ js/qtl/qtlTrack.js | 9 ++++++- js/searchFeatures.js | 20 +++++++++------ test/testSearch.js | 45 +++++++++++++++++++++------------ 4 files changed, 51 insertions(+), 25 deletions(-) diff --git a/js/feature/textFeatureSource.js b/js/feature/textFeatureSource.js index ff6baeef6..6dfe2cec9 100644 --- a/js/feature/textFeatureSource.js +++ b/js/feature/textFeatureSource.js @@ -280,6 +280,8 @@ class TextFeatureSource extends BaseFeatureSource { } catch (error) { console.error(error) this.featureMap = new Map() // Don't retry the load on every search + } finally { + this.searchLoad = undefined } } if (this.featureMap) { diff --git a/js/qtl/qtlTrack.js b/js/qtl/qtlTrack.js index 80e667984..f0c6d4bbe 100755 --- a/js/qtl/qtlTrack.js +++ b/js/qtl/qtlTrack.js @@ -291,7 +291,14 @@ class QTLTrack extends TrackBase { // possibly Expand region to bring phenotype in view const canonicalChrName = this.browser.genome.getChromosomeName(chr) - const features = await Promise.all([...genes].map(term => searchFeatures(this.browser, term))) + // Look up a few genes at a time: all at once would queue requests in the browser + // long enough to time out. A lookup that fails only loses that gene. + const terms = [...genes] + const features = [] + for (let i = 0; i < terms.length; i += 4) { + const lookups = terms.slice(i, i + 4).map(term => searchFeatures(this.browser, term).catch(() => undefined)) + features.push(...await Promise.all(lookups)) + } for (let feature of features) { if (feature) { if (canonicalChrName === this.browser.genome.getChromosomeName(feature.chr)) { diff --git a/js/searchFeatures.js b/js/searchFeatures.js index da60122c0..6a637cefb 100644 --- a/js/searchFeatures.js +++ b/js/searchFeatures.js @@ -19,9 +19,9 @@ const DEFAULT_SEARCH_CONFIG = { const MAX_NOT_FOUND = 100 const notFoundURLs = new Set() -// Services that timed out or could not be reached are skipped for a while, so a search of several terms, or a -// repeated search, waits for one timeout and not one per lookup. Keyed by the service url template; the value is -// the time (ms) at which the service will be tried again. +// A service that timed out is skipped for a while, so a search of several terms, or a repeated search, waits for +// one timeout and not one per lookup. Keyed by the service url template; the value is the time (ms) at which the +// service will be tried again. const SERVICE_RETRY_INTERVAL = 60000 const unavailableServices = new Map() @@ -84,18 +84,22 @@ async function searchWebService(browser, locus, searchConfig) { const retryTime = unavailableServices.get(searchConfig.url) if (retryTime !== undefined) { if (Date.now() < retryTime) { - return undefined + // Thrown, not returned as a miss, so the caller can tell "not searched" from "not found" + throw new Error("the service timed out and is not being tried for now") } unavailableServices.delete(searchConfig.url) } + const timeout = searchConfig.timeout ?? DEFAULT_TIMEOUT + const requestTime = Date.now() let result try { - result = await igvxhr.loadString(path, {timeout: searchConfig.timeout ?? DEFAULT_TIMEOUT}) + result = await igvxhr.loadString(path, {timeout}) } catch (error) { - // Status 0 is a timeout or a service that could not be reached. That says nothing about this term, so - // the miss is not remembered, the service is rested instead. - if (error && error.status === 0) { + // A timeout says nothing about this term, so the miss is not remembered, the service is rested instead. + // A timeout has status 0, but so does a response blocked by CORS, which can be the answer for a single + // term; the time taken tells them apart. + if (error && error.status === 0 && Date.now() - requestTime >= timeout) { unavailableServices.set(searchConfig.url, Date.now() + SERVICE_RETRY_INTERVAL) } throw error diff --git a/test/testSearch.js b/test/testSearch.js index 4cab62c19..13e43f337 100644 --- a/test/testSearch.js +++ b/test/testSearch.js @@ -197,49 +197,62 @@ suite("testSearch", function () { assert.isUndefined(await searchWebService(browser, "nosuchgene", searchConfig)) }) - test("webservice that cannot be reached is rested", async function () { + test("webservice that times out is rested", async function () { - const searchConfig = {type: "plain", url: "https://unreachable.example/a?name=$FEATURE$", coords: 0} + const searchConfig = {type: "plain", url: "https://slow.example/a?name=$FEATURE$", coords: 0, timeout: 100} const loadString = igvxhr.loadString + const now = Date.now + let time = 1000 + Date.now = () => time const requests = [] igvxhr.loadString = async (url, options) => { requests.push({url, options}) + time += options.timeout // The request takes the whole timeout const error = new Error("timeout") error.status = 0 throw error } - try { - let failed = false + const fails = async (term, config = searchConfig) => { try { - await searchWebService(browser, "gene1", searchConfig) + await searchWebService(browser, term, config) + return false } catch (e) { - failed = true + return true } - assert.isTrue(failed) + } + try { + assert.isTrue(await fails("gene1")) + assert.equal(requests[0].options.timeout, 100) - // The service is not tried again for other terms, they are reported as not found - assert.isUndefined(await searchWebService(browser, "gene2", searchConfig)) - assert.isUndefined(await searchWebService(browser, "gene1", searchConfig)) + // The service is not tried again, for this or other terms, and says so by failing + assert.isTrue(await fails("gene2")) + assert.isTrue(await fails("gene1")) assert.equal(requests.length, 1) - // A timeout is always set, the configured one if there is one - assert.isAbove(requests[0].options.timeout, 0) - await searchWebService(browser, "gene1", {...searchConfig, url: "https://unreachable.example/b?name=$FEATURE$", timeout: 123}).catch(() => {}) - assert.equal(requests[1].options.timeout, 123) + // ... until some time has passed + time += 10 * 60 * 1000 + assert.isTrue(await fails("gene2")) + assert.equal(requests.length, 2) + + // A timeout is set when the config has none + await fails("gene1", {type: "plain", url: "https://slow.example/b?name=$FEATURE$", coords: 0}) + assert.isAbove(requests[2].options.timeout, 0) } finally { igvxhr.loadString = loadString + Date.now = now } }) test("webservice error for one term does not rest the service", async function () { + // Status 0 that comes back at once is not a timeout, it could be an error response blocked by CORS const searchConfig = {type: "plain", url: "https://failing.example/?name=$FEATURE$", coords: 0} const loadString = igvxhr.loadString let requests = 0 igvxhr.loadString = async () => { requests++ - const error = new Error("bad request") - error.status = 400 + const error = new Error("blocked") + error.status = 0 throw error } try { From fad6db6a8a57e8d6bdb9c48c11f1fae6361cd3ba Mon Sep 17 00:00:00 2001 From: Jim Robinson <933148+jrobinso@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:12:54 -0500 Subject: [PATCH 3/3] Share the bigBed search index load Co-Authored-By: Claude Fable 5.2 --- js/bigwig/bwReader.js | 19 ++++++++++++++----- js/searchFeatures.js | 4 ++-- test/testBigbed.js | 10 ++++++++++ test/testSearch.js | 6 ++++++ 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/js/bigwig/bwReader.js b/js/bigwig/bwReader.js index 63c3b9203..f8ee7c9dc 100644 --- a/js/bigwig/bwReader.js +++ b/js/bigwig/bwReader.js @@ -283,11 +283,20 @@ class BWReader { if (this._searchTrees === undefined && this.header.extraIndexOffsets && this.header.extraIndexOffsets.length > 0) { - this._searchTrees = [] - for (let offset of this.header.extraIndexOffsets) { - const type = undefined - const bpTree = await BPTree.loadBpTree(this.path, this.config, offset, type, this.indexLoader) - this._searchTrees.push(bpTree) + // One load, shared by searches that arrive while it is in progress. The trees are not exposed until + // all are loaded, a partial list would read as "not found". + this._searchTreesLoad ??= (async () => { + const searchTrees = [] + for (let offset of this.header.extraIndexOffsets) { + const type = undefined + searchTrees.push(await BPTree.loadBpTree(this.path, this.config, offset, type, this.indexLoader)) + } + this._searchTrees = searchTrees + })() + try { + await this._searchTreesLoad + } finally { + this._searchTreesLoad = undefined } } return this._searchTrees diff --git a/js/searchFeatures.js b/js/searchFeatures.js index 6a637cefb..611bc7f3f 100644 --- a/js/searchFeatures.js +++ b/js/searchFeatures.js @@ -98,8 +98,8 @@ async function searchWebService(browser, locus, searchConfig) { } catch (error) { // A timeout says nothing about this term, so the miss is not remembered, the service is rested instead. // A timeout has status 0, but so does a response blocked by CORS, which can be the answer for a single - // term; the time taken tells them apart. - if (error && error.status === 0 && Date.now() - requestTime >= timeout) { + // term; the time taken tells them apart. A timeout of 0 means no timeout. + if (error && error.status === 0 && timeout > 0 && Date.now() - requestTime >= timeout) { unavailableServices.set(searchConfig.url, Date.now() + SERVICE_RETRY_INTERVAL) } throw error diff --git a/test/testBigbed.js b/test/testBigbed.js index 9121c01ca..5d866d5ba 100644 --- a/test/testBigbed.js +++ b/test/testBigbed.js @@ -224,6 +224,16 @@ table chromatinInteract }) + test("concurrent first searches", async function () { + + // Searches that start before the search index is loaded must all wait for it + const bbReader = new BWReader({url: "test/data/bb/GCA_009914755.4.chromAlias.bb", format: "bigbed"}, createGenome()) + const features = await Promise.all(["CP068277.2", "chr1", "CP068276.2", "chr2"].map(term => bbReader.search(term))) + for (const feature of features) { + assert.ok(feature) + } + }) + /** * Test parsing a aliasBbURL file from the T2T hub */ diff --git a/test/testSearch.js b/test/testSearch.js index 13e43f337..983135fc4 100644 --- a/test/testSearch.js +++ b/test/testSearch.js @@ -259,6 +259,12 @@ suite("testSearch", function () { await searchWebService(browser, "gene1", searchConfig).catch(() => {}) await searchWebService(browser, "gene2", searchConfig).catch(() => {}) assert.equal(requests, 2) + + // Nor with the timeout turned off + const noTimeout = {...searchConfig, url: "https://failing.example/b?name=$FEATURE$", timeout: 0} + await searchWebService(browser, "gene1", noTimeout).catch(() => {}) + await searchWebService(browser, "gene2", noTimeout).catch(() => {}) + assert.equal(requests, 4) } finally { igvxhr.loadString = loadString }