Skip to content

Commit 3a85402

Browse files
committed
Preserve bootstrap ownership and avoid repeated startup analysis
1 parent fe2961b commit 3a85402

7 files changed

Lines changed: 199 additions & 25 deletions

File tree

‎crates/common/ARCHITECTURE.md‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -146,17 +146,21 @@ Target gating is used extensively:
146146
the sync scheduler.
147147
- Browser startup starts fully warm networks independently, then coordinates
148148
missing targets under a shared Web Lock and their per-network leadership
149-
locks. Networks led in another tab are left to their existing runners. Seeds
150-
download in parallel; failed downloads leave only those targets for retry,
151-
while successful downloads are rechecked against persisted watermarks and
152-
imported together in one bounded sqlite-web session. A globally fresh DB drops
149+
locks. Ownership transfers directly to cold runners, including on failure;
150+
networks led in another tab are left to their existing runners. Seeds download
151+
in parallel; failed downloads leave only those targets for retry, while
152+
successful downloads are rechecked against persisted watermarks and imported
153+
together in one bounded sqlite-web session. A globally fresh DB drops
153154
non-unique secondary indexes and rebuilds them once after all seeds; additions
154-
to a populated DB retain existing indexes. PRIMARY KEY/UNIQUE indexes remain
155-
enforced. One best-effort `ANALYZE` follows, then cold network loops catch up
156-
and establish readiness. Import/manifest failures and unsupported coordination
157-
fall back to per-network provisioning; stale-target refresh retains its
158-
existing behavior. Remote waits are bounded to two minutes per request and
159-
scheduler stop cancels coordination/import cleanup.
155+
to a populated DB retain existing indexes. A shared import Web Lock covers
156+
watermark/index preflight through commit or cancellation for both cohort and
157+
per-network imports; without that ownership, or after SDK lease contention,
158+
imports retain indexes. PRIMARY KEY/UNIQUE indexes remain enforced. Cold
159+
network loops catch up, analyze their applied data and establish readiness;
160+
the cohort does not run an additional `ANALYZE`. Import/manifest failures and
161+
unsupported coordination fall back to per-network provisioning; stale-target
162+
refresh retains its existing behavior. Remote waits are bounded to two minutes
163+
per request and scheduler stop cancels coordination/import cleanup.
160164
- `LocalDbState` encapsulates the local DB handle, scheduler, `SyncReadiness`
161165
(tracks which chains have completed a sync cycle), and the set of configured
162166
chain IDs. Query routing uses `QuerySource::LocalDb` vs

‎crates/common/src/raindex_client/local_db/executor.rs‎

Lines changed: 68 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,7 @@ struct ImportSessionGuard {
232232
executor: JsCallbackExecutor,
233233
pending: Option<PendingImport>,
234234
serialize: Option<OwnedMutexGuard<()>>,
235+
ownership: Option<super::pipeline::runner::leadership::LeadershipGuard>,
235236
}
236237

237238
enum PendingImport {
@@ -275,9 +276,11 @@ impl Drop for ImportSessionGuard {
275276
};
276277
let executor = self.executor.clone();
277278
let serialize = self.serialize.take();
279+
let ownership = self.ownership.take();
278280
wasm_bindgen_utils::prelude::wasm_bindgen_futures::spawn_local(async move {
279281
// Keep later queries/imports serialized until rollback finishes.
280282
let _serialize = serialize;
283+
let _ownership = ownership;
281284
let id = match pending {
282285
PendingImport::Active { id, in_flight } => {
283286
// The SDK rejects cancellation while an append/finish/cancel
@@ -410,6 +413,29 @@ impl JsCallbackExecutor {
410413
})
411414
})
412415
.collect::<Result<Vec<_>, _>>()?;
416+
// Both cohort and per-network imports hold the same cross-tab lock
417+
// before reading watermarks. A competing import cannot populate the DB
418+
// between this preflight and opening our worker session.
419+
let started = js_sys::Date::now();
420+
let ownership = loop {
421+
match super::pipeline::runner::leadership::acquire_import().await {
422+
Ok(Some(guard)) => break Some(guard),
423+
Ok(None) if within_import_wait(started) => {
424+
gloo_timers::future::TimeoutFuture::new(IMPORT_WAIT_POLL_MS).await;
425+
}
426+
Ok(None) => {
427+
return Err(LocalDbQueryError::database(
428+
"Timed out waiting for import ownership",
429+
));
430+
}
431+
Err(error) => {
432+
// Import still works through the SDK lease without Web
433+
// Locks, but preflight cannot safely justify index drops.
434+
tracing::warn!(%error, "Import ownership unavailable; retaining indexes");
435+
break None;
436+
}
437+
}
438+
};
413439
// Only a globally fresh DB benefits from rebuilding every index. Adding
414440
// seeds to a populated DB keeps indexes over its existing history intact.
415441
let watermarks = self
@@ -419,7 +445,7 @@ impl JsCallbackExecutor {
419445
.await?;
420446
let has_watermarks: Vec<serde_json::Value> =
421447
serde_json::from_str(&watermarks).map_err(|_| LocalDbQueryError::invalid_response())?;
422-
let defer_indexes = has_watermarks.is_empty();
448+
let defer_indexes = ownership.is_some() && has_watermarks.is_empty();
423449
let mut prefix = String::from("BEGIN;\n");
424450
let mut suffix = String::new();
425451
if defer_indexes {
@@ -445,6 +471,7 @@ impl JsCallbackExecutor {
445471
executor: self.clone(),
446472
pending: None,
447473
serialize: Some(serialize),
474+
ownership,
448475
};
449476
let started = js_sys::Date::now();
450477
let id = loop {
@@ -457,6 +484,11 @@ impl JsCallbackExecutor {
457484
match result {
458485
Err(err) if err.is_import_in_progress() && within_import_wait(started) => {
459486
guard.pending = None;
487+
// An older SDK caller may not participate in our Web Lock.
488+
// Its import invalidates preflight; retain indexes rather
489+
// than rebuild over data committed while we waited.
490+
prefix = String::from("BEGIN;\n");
491+
suffix = String::from("COMMIT;\n");
460492
gloo_timers::future::TimeoutFuture::new(IMPORT_WAIT_POLL_MS).await;
461493
}
462494
result => break result?,
@@ -1318,6 +1350,41 @@ pub mod tests {
13181350
}
13191351
}
13201352

1353+
#[wasm_bindgen_test]
1354+
async fn import_lease_contention_retains_preflight_indexes() {
1355+
let local_db = Function::new_no_args(r#"return {
1356+
chunks: [], begins: 0,
1357+
query(sql) { return {value: sql.includes('target_watermarks') ? '[]' :
1358+
'[{"name":"idx_t_value","sql":"CREATE INDEX idx_t_value ON t(value)"}]'}; },
1359+
transaction() {}, wipeAndRecreate() {},
1360+
beginSqlDumpImport() {
1361+
if (++this.begins === 1) return {error:{msg:'busy',readableMsg:'SQL dump import is in progress'}};
1362+
return {value:'session'};
1363+
},
1364+
appendSqlDumpChunk(id, sql) { this.chunks.push(sql); return {value:''}; },
1365+
finishSqlDumpImport() { return {value:''}; }, cancelSqlDumpImport() {}
1366+
};"#).call0(&JsValue::UNDEFINED).unwrap();
1367+
let exec = JsCallbackExecutor::new(local_db.clone()).unwrap();
1368+
exec.execute_sql_dump(Arc::new("BEGIN; INSERT INTO t VALUES (1); COMMIT;".into()))
1369+
.await
1370+
.unwrap();
1371+
let chunks = Reflect::get(&local_db, &"chunks".into())
1372+
.unwrap()
1373+
.dyn_into::<Array>()
1374+
.unwrap();
1375+
let sql = chunks
1376+
.iter()
1377+
.map(|v| v.as_string().unwrap())
1378+
.collect::<String>();
1379+
assert!(sql.contains("INSERT INTO t VALUES (1)"));
1380+
assert!(!sql.contains("DROP INDEX"));
1381+
assert!(!sql.contains("CREATE INDEX"));
1382+
assert_eq!(
1383+
Reflect::get(&local_db, &"begins".into()).unwrap().as_f64(),
1384+
Some(2.0)
1385+
);
1386+
}
1387+
13211388
#[wasm_bindgen_test]
13221389
async fn cohort_rejects_bad_envelopes_before_opening_a_session() {
13231390
let local_db = Function::new_no_args(

‎crates/common/src/raindex_client/local_db/mod.rs‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -416,6 +416,62 @@ mod wasm_tests {
416416

417417
wasm_bindgen_test_configure!(run_in_browser);
418418

419+
#[wasm_bindgen_test]
420+
async fn import_preflight_waits_for_cross_tab_ownership() {
421+
use super::pipeline::runner::leadership::acquire_import;
422+
use js_sys::{Array, Function, Reflect};
423+
use std::sync::Arc;
424+
let mut holder = None;
425+
for _ in 0..100 {
426+
holder = acquire_import().await.unwrap();
427+
if holder.is_some() {
428+
break;
429+
}
430+
TimeoutFuture::new(5).await;
431+
}
432+
let holder = holder.expect("previous test must release import ownership");
433+
let local_db = Function::new_no_args(r#"return {
434+
reads:0, populated:false, chunks:[],
435+
query(sql) {
436+
this.reads++;
437+
if (!sql.includes('target_watermarks')) throw new Error('must retain populated indexes');
438+
return {value:this.populated ? '[{"present":1}]' : '[]'};
439+
},
440+
transaction() {}, wipeAndRecreate() {},
441+
beginSqlDumpImport() { return {value:'session'}; },
442+
appendSqlDumpChunk(id, sql) { this.chunks.push(sql); return {value:''}; },
443+
finishSqlDumpImport() { return {value:''}; }, cancelSqlDumpImport() {}
444+
};"#).call0(&JsValue::UNDEFINED).unwrap();
445+
let exec = JsCallbackExecutor::new(local_db.clone()).unwrap();
446+
let mut import = exec.execute_sql_dumps(vec![Arc::new(
447+
"BEGIN; INSERT INTO t VALUES (2); COMMIT;".into(),
448+
)]);
449+
for _ in 0..10 {
450+
assert!(futures::poll!(import.as_mut()).is_pending());
451+
gloo_timers::future::TimeoutFuture::new(1).await;
452+
}
453+
assert_eq!(
454+
Reflect::get(&local_db, &"reads".into()).unwrap().as_f64(),
455+
Some(0.0)
456+
);
457+
Reflect::set(&local_db, &"populated".into(), &JsValue::TRUE).unwrap();
458+
drop(holder);
459+
import.await.unwrap();
460+
assert_eq!(
461+
Reflect::get(&local_db, &"reads".into()).unwrap().as_f64(),
462+
Some(1.0)
463+
);
464+
let chunks = Reflect::get(&local_db, &"chunks".into())
465+
.unwrap()
466+
.dyn_into::<Array>()
467+
.unwrap();
468+
let sql = chunks
469+
.iter()
470+
.map(|v| v.as_string().unwrap())
471+
.collect::<String>();
472+
assert!(!sql.contains("DROP INDEX"));
473+
}
474+
419475
fn single_raindex_settings_yaml() -> String {
420476
format!(
421477
r#"

‎crates/common/src/raindex_client/local_db/pipeline/runner/bootstrap.rs‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -139,9 +139,8 @@ where
139139
db.execute_sql_dumps(dumps)
140140
.await
141141
.map_err(LocalDbError::DumpImportFailed)?;
142-
if let Err(error) = db.query_text(&SqlStatement::new("ANALYZE")).await {
143-
tracing::warn!(%error, "Coordinated bootstrap ANALYZE failed");
144-
}
142+
// The normal first catch-up cycle analyzes its applied data. An ANALYZE
143+
// here would repeat that full-database scan before the networks are ready.
145144
Ok(())
146145
}
147146

@@ -345,7 +344,7 @@ local-db-sync:
345344
assert_eq!(imports[0].len(), 2);
346345
assert!(imports[0][0].contains("'a'"));
347346
assert!(imports[0][1].contains("'b'"));
348-
assert_eq!(*db.analyzes.borrow(), 1);
347+
assert_eq!(*db.analyzes.borrow(), 0, "catch-up owns ANALYZE");
349348
}
350349
}
351350

@@ -369,7 +368,7 @@ local-db-sync:
369368
let imports = db.imports.borrow();
370369
assert_eq!(imports.len(), 1);
371370
assert_eq!(imports[0].len(), 2);
372-
assert_eq!(*db.analyzes.borrow(), 1);
371+
assert_eq!(*db.analyzes.borrow(), 0, "catch-up owns ANALYZE");
373372
}
374373

375374
#[wasm_bindgen_test]
@@ -445,7 +444,7 @@ local-db-sync:
445444
assert_eq!(imports.len(), 1);
446445
assert_eq!(imports[0].len(), 1);
447446
assert!(imports[0][0].contains("'a'"));
448-
assert_eq!(*db.analyzes.borrow(), 1);
447+
assert_eq!(*db.analyzes.borrow(), 0, "catch-up owns ANALYZE");
449448
}
450449

451450
#[wasm_bindgen_test]

‎crates/common/src/raindex_client/local_db/pipeline/runner/leadership.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,16 @@ pub(crate) async fn acquire_bootstrap() -> Result<Option<LeadershipGuard>, Local
117117
acquire_bootstrap_lock("local-db-bootstrap").await
118118
}
119119

120+
/// Serialize browser dump preflight and import, including per-network fallback.
121+
#[cfg(target_family = "wasm")]
122+
pub(crate) async fn acquire_import() -> Result<Option<LeadershipGuard>, LocalDbError> {
123+
// Non-browser Wasm callers have no cross-tab ownership to coordinate.
124+
if window().is_none() {
125+
return Ok(Some(LeadershipGuard::new_noop()));
126+
}
127+
acquire_bootstrap_lock("local-db-sql-dump-import").await
128+
}
129+
120130
/// Protect initial seed writes from existing per-network leaders in other tabs.
121131
#[cfg(target_family = "wasm")]
122132
pub(crate) async fn acquire_network_bootstrap(

‎crates/common/src/raindex_client/local_db/pipeline/runner/mod.rs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,13 @@ where
136136
!self.manifests_loaded || !self.has_provisioned_dumps
137137
}
138138

139+
/// Keep the coordinator's network ownership through the first sync cycle.
140+
#[cfg(any(target_family = "wasm", test))]
141+
pub(crate) fn with_initial_leadership(mut self, guard: Option<LeadershipGuard>) -> Self {
142+
self.leadership_guard = guard;
143+
self
144+
}
145+
139146
pub async fn run<DB>(&mut self, db: &DB) -> Result<RunOutcome, LocalDbError>
140147
where
141148
DB: LocalDbQueryExecutor + ?Sized,
@@ -1520,6 +1527,29 @@ raindexes:
15201527
assert_eq!(telemetry.builder_inits(), 4);
15211528
}
15221529

1530+
#[tokio::test]
1531+
async fn transferred_leadership_skips_reacquisition() {
1532+
let telemetry = Telemetry::default();
1533+
let environment = build_environment(manifest_for_both(), HashMap::new(), 1, 2, telemetry);
1534+
// No acquire is allowed: the coordinator already owns this network,
1535+
// even if one of its seed downloads failed and the runner must retry.
1536+
let leadership = SequenceLeadership::new(Vec::new());
1537+
let mut runner =
1538+
ClientRunner::with_environment(two_raindexes_settings_yaml(), environment, leadership)
1539+
.unwrap()
1540+
.with_initial_leadership(Some(LeadershipGuard::new_noop()));
1541+
let db = RecordingDb::default();
1542+
prepare_db_for_targets(&db, &runner.base_targets);
1543+
assert!(matches!(
1544+
runner.run(&db).await.unwrap(),
1545+
RunOutcome::Report(_)
1546+
));
1547+
assert!(matches!(
1548+
runner.run(&db).await.unwrap(),
1549+
RunOutcome::Report(_)
1550+
));
1551+
}
1552+
15231553
#[tokio::test]
15241554
async fn leadership_guard_reused_across_runs() {
15251555
let telemetry = Telemetry::default();

‎crates/common/src/raindex_client/local_db/pipeline/runner/scheduler/wasm.rs‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use super::super::bootstrap::{provision_missing_targets, warm_network_keys};
22
use super::super::config::NetworkRunnerConfig;
33
use super::super::environment::default_environment;
4-
use super::super::leadership::DefaultLeadership;
4+
use super::super::leadership::{DefaultLeadership, LeadershipGuard};
55
use super::super::ClientRunner;
66
use crate::local_db::pipeline::adapters::bootstrap::BootstrapPipeline;
77
use crate::local_db::pipeline::adapters::{
@@ -31,7 +31,7 @@ use js_sys::Function;
3131
use raindex_app_settings::local_db_manifest::DB_SCHEMA_VERSION;
3232
use raindex_app_settings::network::NetworkCfg;
3333
use std::cell::Cell;
34-
use std::collections::HashSet;
34+
use std::collections::{HashMap, HashSet};
3535
use std::future::Future;
3636
use std::pin::Pin;
3737
use std::rc::Rc;
@@ -151,7 +151,7 @@ pub(crate) fn start(
151151
}
152152
}
153153

154-
let spawn_network = |network: &NetworkCfg| {
154+
let spawn_network = |network: &NetworkCfg, ownership: Option<LeadershipGuard>| {
155155
let config =
156156
match NetworkRunnerConfig::from_global_settings(&settings_clone, &network.key) {
157157
Ok(config) => config,
@@ -169,7 +169,7 @@ pub(crate) fn start(
169169
let environment = default_environment(status_store.clone());
170170

171171
let runner = match ClientRunner::from_config(config.clone(), environment, leadership) {
172-
Ok(r) => r,
172+
Ok(r) => r.with_initial_leadership(ownership),
173173
Err(err) => {
174174
emit_network_status(
175175
&status_store,
@@ -220,10 +220,19 @@ pub(crate) fn start(
220220
}
221221
for network in &networks_for_spawn {
222222
if warm.contains(&network.key) {
223-
spawn_network(network);
223+
spawn_network(network, None);
224224
}
225225
}
226+
if networks_for_spawn
227+
.iter()
228+
.all(|network| warm.contains(&network.key))
229+
{
230+
return;
231+
}
226232

233+
// Retain guards even on provisioning failure, and hand them directly to
234+
// the runners. Dropping/reacquiring Web Locks has an asynchronous gap.
235+
let mut network_ownership = HashMap::new();
227236
let provisioning = async {
228237
let _ownership = loop {
229238
if let Some(guard) = super::super::leadership::acquire_bootstrap().await? {
@@ -235,7 +244,6 @@ pub(crate) fn start(
235244
// fallback/RPC leaders from writing the same target in another tab.
236245
let mut cohort_settings = settings_clone.clone();
237246
cohort_settings.syncs.clear();
238-
let mut network_ownership = Vec::new();
239247
for network in &networks_for_spawn {
240248
if warm.contains(&network.key) {
241249
continue;
@@ -247,7 +255,7 @@ pub(crate) fn start(
247255
cohort_settings
248256
.syncs
249257
.insert(network.key.clone(), sync.clone());
250-
network_ownership.push(guard);
258+
network_ownership.insert(network.key.clone(), guard);
251259
}
252260
}
253261
}
@@ -280,7 +288,7 @@ pub(crate) fn start(
280288

281289
for network in &networks_for_spawn {
282290
if !warm.contains(&network.key) {
283-
spawn_network(network);
291+
spawn_network(network, network_ownership.remove(&network.key));
284292
}
285293
}
286294
});

0 commit comments

Comments
 (0)