Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions js/bigwig/bwReader.js
Original file line number Diff line number Diff line change
Expand Up @@ -283,11 +283,20 @@ class BWReader {
if (this._searchTrees === undefined &&
this.header.extraIndexOffsets &&
this.header.extraIndexOffsets.length > 0) {
this._searchTrees = []
for (let offset of this.header.extraIndexOffsets) {
const type = undefined
const bpTree = await BPTree.loadBpTree(this.path, this.config, offset, type, this.indexLoader)
this._searchTrees.push(bpTree)
// One load, shared by searches that arrive while it is in progress. The trees are not exposed until
// all are loaded, a partial list would read as "not found".
this._searchTreesLoad ??= (async () => {
const searchTrees = []
for (let offset of this.header.extraIndexOffsets) {
const type = undefined
searchTrees.push(await BPTree.loadBpTree(this.path, this.config, offset, type, this.indexLoader))
}
this._searchTrees = searchTrees
})()
try {
await this._searchTreesLoad
} finally {
this._searchTreesLoad = undefined
}
}
return this._searchTrees
Expand Down
3 changes: 2 additions & 1 deletion js/browser.js
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,8 @@ class Browser {
endField: config.search.endField || "end",
geneField: config.search.geneField || "gene",
snpField: config.search.snpField || "snp",
resultsField: config.search.resultsField
resultsField: config.search.resultsField,
timeout: config.search.timeout
}
}
}
Expand Down
6 changes: 5 additions & 1 deletion js/feature/textFeatureSource.js
Original file line number Diff line number Diff line change
Expand Up @@ -274,10 +274,14 @@ class TextFeatureSource extends BaseFeatureSource {
// Any chromosome will do, the whole file is read. chromosomeNames is optional, fall back to a loaded chromosome.
const chr = this.genome.chromosomeNames?.[0] ?? this.genome.chromosomes.keys().next().value
try {
await this.getFeatures({chr, start: 0, end: Number.MAX_SAFE_INTEGER})
// One load, shared by searches that arrive while it is in progress
this.searchLoad ??= this.getFeatures({chr, start: 0, end: Number.MAX_SAFE_INTEGER})
await this.searchLoad
} catch (error) {
console.error(error)
this.featureMap = new Map() // Don't retry the load on every search
} finally {
this.searchLoad = undefined
}
}
if (this.featureMap) {
Expand Down
1 change: 1 addition & 0 deletions js/igv.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -508,6 +508,7 @@ interface CreateOptExtras {

search?: {
url: string;
timeout?: number;
coords?: 0 | 1;
chromosomeField?: string;
startField?: string;
Expand Down
11 changes: 9 additions & 2 deletions js/qtl/qtlTrack.js
Original file line number Diff line number Diff line change
Expand Up @@ -291,8 +291,15 @@ class QTLTrack extends TrackBase {

// possibly Expand region to bring phenotype in view
const canonicalChrName = this.browser.genome.getChromosomeName(chr)
for (let term of genes) {
const feature = await searchFeatures(this.browser, term)
// Look up a few genes at a time: all at once would queue requests in the browser
// long enough to time out. A lookup that fails only loses that gene.
const terms = [...genes]
const features = []
for (let i = 0; i < terms.length; i += 4) {
const lookups = terms.slice(i, i + 4).map(term => searchFeatures(this.browser, term).catch(() => undefined))
features.push(...await Promise.all(lookups))
}
for (let feature of features) {
if (feature) {
if (canonicalChrName === this.browser.genome.getChromosomeName(feature.chr)) {
start = Math.min(start, feature.start)
Expand Down
35 changes: 32 additions & 3 deletions js/searchFeatures.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@ import {igvxhr, StringUtils} from "../node_modules/igv-utils/src/index.js"
* This module is separate to avoid circular dependencies between search.js and hgvs.js
*/

// Applied to any search service whose config does not set a timeout, so a hung service cannot block a search
const DEFAULT_TIMEOUT = 5000

const DEFAULT_SEARCH_CONFIG = {
timeout: 2000,
type: "plain",
url: 'https://igv.org/genomes/locus.php?genome=$GENOME$&name=$FEATURE$',
coords: 0
Expand All @@ -17,6 +19,12 @@ const DEFAULT_SEARCH_CONFIG = {
const MAX_NOT_FOUND = 100
const notFoundURLs = new Set()

// A service that timed out is skipped for a while, so a search of several terms, or a repeated search, waits for
// one timeout and not one per lookup. Keyed by the service url template; the value is the time (ms) at which the
// service will be tried again.
const SERVICE_RETRY_INTERVAL = 60000
const unavailableServices = new Map()

/**
* Search for a feature by name in MANE transcripts, searchable tracks, and web services
* @param {Object} browser - The IGV browser instance
Expand Down Expand Up @@ -73,8 +81,29 @@ async function searchWebService(browser, locus, searchConfig) {
return undefined
}

const options = searchConfig.timeout ? {timeout: searchConfig.timeout} : undefined
const result = await igvxhr.loadString(path, options)
const retryTime = unavailableServices.get(searchConfig.url)
if (retryTime !== undefined) {
if (Date.now() < retryTime) {
// Thrown, not returned as a miss, so the caller can tell "not searched" from "not found"
throw new Error("the service timed out and is not being tried for now")
}
unavailableServices.delete(searchConfig.url)
}

const timeout = searchConfig.timeout ?? DEFAULT_TIMEOUT
const requestTime = Date.now()
let result
try {
result = await igvxhr.loadString(path, {timeout})
} catch (error) {
// A timeout says nothing about this term, so the miss is not remembered, the service is rested instead.
// A timeout has status 0, but so does a response blocked by CORS, which can be the answer for a single
// term; the time taken tells them apart. A timeout of 0 means no timeout.
if (error && error.status === 0 && timeout > 0 && Date.now() - requestTime >= timeout) {
unavailableServices.set(searchConfig.url, Date.now() + SERVICE_RETRY_INTERVAL)
}
throw error
}

const feature = await processSearchResult(browser, result, searchConfig)
if (!feature) {
Expand Down
10 changes: 10 additions & 0 deletions test/testBigbed.js
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,16 @@ table chromatinInteract

})

test("concurrent first searches", async function () {

// Searches that start before the search index is loaded must all wait for it
const bbReader = new BWReader({url: "test/data/bb/GCA_009914755.4.chromAlias.bb", format: "bigbed"}, createGenome())
const features = await Promise.all(["CP068277.2", "chr1", "CP068276.2", "chr2"].map(term => bbReader.search(term)))
for (const feature of features) {
assert.ok(feature)
}
})

/**
* Test parsing a aliasBbURL file from the T2T hub
*/
Expand Down
74 changes: 74 additions & 0 deletions test/testSearch.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {createGenome} from "./utils/MockGenome.js"
import search, {parseLocusString, searchWebService} from "../js/search.js"
import FeatureSource from "../js/feature/featureSource.js"
import {searchFeatures} from "../js/searchFeatures.js"
import {igvxhr} from "../node_modules/igv-utils/src/index.js"

const genome = createGenome()

Expand Down Expand Up @@ -196,6 +197,79 @@ suite("testSearch", function () {
assert.isUndefined(await searchWebService(browser, "nosuchgene", searchConfig))
})

test("webservice that times out is rested", async function () {

const searchConfig = {type: "plain", url: "https://slow.example/a?name=$FEATURE$", coords: 0, timeout: 100}
const loadString = igvxhr.loadString
const now = Date.now
let time = 1000
Date.now = () => time
const requests = []
igvxhr.loadString = async (url, options) => {
requests.push({url, options})
time += options.timeout // The request takes the whole timeout
const error = new Error("timeout")
error.status = 0
throw error
}
const fails = async (term, config = searchConfig) => {
try {
await searchWebService(browser, term, config)
return false
} catch (e) {
return true
}
}
try {
assert.isTrue(await fails("gene1"))
assert.equal(requests[0].options.timeout, 100)

// The service is not tried again, for this or other terms, and says so by failing
assert.isTrue(await fails("gene2"))
assert.isTrue(await fails("gene1"))
assert.equal(requests.length, 1)

// ... until some time has passed
time += 10 * 60 * 1000
assert.isTrue(await fails("gene2"))
assert.equal(requests.length, 2)

// A timeout is set when the config has none
await fails("gene1", {type: "plain", url: "https://slow.example/b?name=$FEATURE$", coords: 0})
assert.isAbove(requests[2].options.timeout, 0)
} finally {
igvxhr.loadString = loadString
Date.now = now
}
})

test("webservice error for one term does not rest the service", async function () {

// Status 0 that comes back at once is not a timeout, it could be an error response blocked by CORS
const searchConfig = {type: "plain", url: "https://failing.example/?name=$FEATURE$", coords: 0}
const loadString = igvxhr.loadString
let requests = 0
igvxhr.loadString = async () => {
requests++
const error = new Error("blocked")
error.status = 0
throw error
}
try {
await searchWebService(browser, "gene1", searchConfig).catch(() => {})
await searchWebService(browser, "gene2", searchConfig).catch(() => {})
assert.equal(requests, 2)

// Nor with the timeout turned off
const noTimeout = {...searchConfig, url: "https://failing.example/b?name=$FEATURE$", timeout: 0}
await searchWebService(browser, "gene1", noTimeout).catch(() => {})
await searchWebService(browser, "gene2", noTimeout).catch(() => {})
assert.equal(requests, 4)
} finally {
igvxhr.loadString = loadString
}
})

test("search non-indexed searchable track, genome without chromosomeNames", async function () {

// Genome with no chromosome list, only the chromosome loaded for the initial locus
Expand Down
Loading