Skip to content

Commit 1e8019b

Browse files
findolorfindolor
committed
Stop writing unused local DB event rows (#2887)
## Motivation Raw RPC logs and signed take-order context rows are stored during local DB sync even though the order and vault read paths do not use them. They add write work and enlarge the producer's data-only dump and browser database. Part of RAI-2648. ## Solution - Stop building `raw_events`, `take_order_contexts`, and `context_values` inserts in the shared apply path used by the producer and browser. - Exclude those three tables from newly exported SQL dumps. - Update tests to assert that take orders are still written, skipped rows are omitted from exports, and producer export fixtures use retained data. The existing table definitions remain. Publishing a newly filtered dump is part of the separate producer rollout. ## Checks - `nix develop .#rust-shell -c cargo test -p raindex_common local_db:: --lib --quiet` — 575 passed. - `nix develop .#rust-shell -c cargo test -p raindex_cli local_db:: --lib --quiet` — 53 passed. - `nix develop .#rust-shell -c cargo fmt --all -- --check` — passed. - `nix develop .#rust-shell -c cargo clippy -p raindex_common -p raindex_cli --lib -- -D warnings` — passed. - CI's `nix develop .#wasm-shell` Wasm test command — passed; runner reported no runnable tests for the four packages on this host. - Two read-only local Codex reviews and a CodeRabbit full review — no actionable findings. Review focus: confirm no retained query or producer path depends on those three row types. The local workspace has unrelated unstaged benchmarking edits; they are not in this PR. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **Data Handling** * Local database exports no longer include raw event records, take-order contexts, or context values. * Event processing no longer saves raw event records or signed take-order context data. It continues to save take-order information and other decoded event data. * Local database export checks now verify that excluded records are absent, and pipeline checks cover deposit data and watermark handling. These changes keep exported and stored data focused on supported decoded information. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
1 parent 4e90291 commit 1e8019b

5 files changed

Lines changed: 46 additions & 462 deletions

File tree

‎crates/cli/src/commands/local_db/pipeline/runner/export.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -195,8 +195,8 @@ mod tests {
195195
decoder.read_to_string(&mut sql).expect("decode gzip");
196196

197197
assert!(
198-
sql.contains("INSERT INTO \"raw_events\""),
199-
"export should include raw_events data"
198+
!sql.contains("INSERT INTO \"raw_events\""),
199+
"export should omit raw_events data"
200200
);
201201
assert!(sql.starts_with("BEGIN;"));
202202
assert!(sql.ends_with("COMMIT;\n"));

‎crates/cli/src/commands/local_db/pipeline/runner/mod.rs‎

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -441,17 +441,18 @@ mod tests {
441441
);",
442442
));
443443
batch.add(SqlStatement::new(
444-
"CREATE TABLE IF NOT EXISTS raw_events (
444+
"CREATE TABLE IF NOT EXISTS deposits (
445445
chain_id INTEGER NOT NULL,
446446
raindex_address TEXT NOT NULL,
447447
transaction_hash TEXT NOT NULL,
448448
log_index INTEGER NOT NULL,
449449
block_number INTEGER NOT NULL,
450-
block_timestamp INTEGER,
451-
address TEXT NOT NULL,
452-
topics TEXT NOT NULL,
453-
data TEXT NOT NULL,
454-
raw_json TEXT NOT NULL,
450+
block_timestamp INTEGER NOT NULL,
451+
sender TEXT NOT NULL,
452+
token TEXT NOT NULL,
453+
vault_id TEXT NOT NULL,
454+
deposit_amount TEXT NOT NULL,
455+
deposit_amount_uint256 TEXT NOT NULL,
455456
PRIMARY KEY (chain_id, raindex_address, transaction_hash, log_index)
456457
);",
457458
));
@@ -535,15 +536,15 @@ mod tests {
535536
{
536537
self.ensure_tables(db).await?;
537538

538-
// Seed a raw event to force export_data_only to return Some, but skip watermark to trigger export failure.
539+
// Seed an exported row but skip the watermark to trigger export failure.
539540
let raindex_id = &config.raindex_id;
540541
let raindex_address = encode_prefixed(raindex_id.raindex_address);
541542
let mut batch = SqlStatementBatch::new();
542543
batch.add(SqlStatement::new(format!(
543-
"INSERT INTO raw_events (chain_id, raindex_address, transaction_hash, log_index, block_number, block_timestamp, address, topics, data, raw_json) \
544-
VALUES ({}, '{}', '0xseedtx', 0, {}, 1_700_000_000, '{}', '[]', '0x00', '{{}}') \
544+
"INSERT INTO deposits (chain_id, raindex_address, transaction_hash, log_index, block_number, block_timestamp, sender, token, vault_id, deposit_amount, deposit_amount_uint256) \
545+
VALUES ({}, '{}', '0xseedtx', 0, {}, 1700000000, '{}', '{}', '1', '1', '1') \
545546
ON CONFLICT(chain_id, raindex_address, transaction_hash, log_index) DO NOTHING;",
546-
raindex_id.chain_id, raindex_address, config.latest_block, raindex_address
547+
raindex_id.chain_id, raindex_address, config.latest_block, raindex_address, raindex_address
547548
)));
548549
db.execute_batch(&batch.ensure_transaction()).await?;
549550

@@ -596,17 +597,18 @@ mod tests {
596597
);",
597598
));
598599
batch.add(SqlStatement::new(
599-
"CREATE TABLE IF NOT EXISTS raw_events (
600+
"CREATE TABLE IF NOT EXISTS deposits (
600601
chain_id INTEGER NOT NULL,
601602
raindex_address TEXT NOT NULL,
602603
transaction_hash TEXT NOT NULL,
603604
log_index INTEGER NOT NULL,
604605
block_number INTEGER NOT NULL,
605-
block_timestamp INTEGER,
606-
address TEXT NOT NULL,
607-
topics TEXT NOT NULL,
608-
data TEXT NOT NULL,
609-
raw_json TEXT NOT NULL,
606+
block_timestamp INTEGER NOT NULL,
607+
sender TEXT NOT NULL,
608+
token TEXT NOT NULL,
609+
vault_id TEXT NOT NULL,
610+
deposit_amount TEXT NOT NULL,
611+
deposit_amount_uint256 TEXT NOT NULL,
610612
PRIMARY KEY (chain_id, raindex_address, transaction_hash, log_index)
611613
);",
612614
));
@@ -702,14 +704,14 @@ mod tests {
702704

703705
let mut batch = SqlStatementBatch::new();
704706
batch.add(SqlStatement::new(format!(
705-
"INSERT INTO raw_events (chain_id, raindex_address, transaction_hash, log_index, block_number, block_timestamp, address, topics, data, raw_json) \
706-
VALUES ({}, '{}', '0xseedtx', 0, {}, 1_700_000_000, '{}', '[]', '0x00', '{{}}') \
707+
"INSERT INTO deposits (chain_id, raindex_address, transaction_hash, log_index, block_number, block_timestamp, sender, token, vault_id, deposit_amount, deposit_amount_uint256) \
708+
VALUES ({}, '{}', '0xseedtx', 0, {}, 1700000000, '{}', '{}', '1', '1', '1') \
707709
ON CONFLICT(chain_id, raindex_address, transaction_hash, log_index) DO NOTHING;",
708-
raindex_id.chain_id, raindex_address, config.latest_block, raindex_address
710+
raindex_id.chain_id, raindex_address, config.latest_block, raindex_address, raindex_address
709711
)));
710712
batch.add(SqlStatement::new(format!(
711713
"INSERT INTO target_watermarks (chain_id, raindex_address, last_block, last_hash, updated_at) \
712-
VALUES ({}, '{}', {}, '0xfeedface', 1_700_000_000_000) \
714+
VALUES ({}, '{}', {}, '0xfeedface', 1700000000000) \
713715
ON CONFLICT(chain_id, raindex_address) DO UPDATE \
714716
SET last_block = excluded.last_block, \
715717
last_hash = excluded.last_hash, \

‎crates/common/src/local_db/export.rs‎

Lines changed: 7 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,13 @@ use thiserror::Error;
1010

1111
use super::RaindexIdentifier;
1212

13-
const SKIPPED_TABLES: &[&str] = &["db_metadata", "sync_status"];
13+
const SKIPPED_TABLES: &[&str] = &[
14+
"db_metadata",
15+
"sync_status",
16+
"raw_events",
17+
"take_order_contexts",
18+
"context_values",
19+
];
1420

1521
#[derive(Debug, Deserialize)]
1622
struct TableInfoRow {
@@ -1260,20 +1266,6 @@ mod tests {
12601266
watermark_ms
12611267
));
12621268

1263-
out.push_str(&format!(
1264-
"INSERT INTO \"raw_events\" (\"chain_id\", \"raindex_address\", \"transaction_hash\", \"log_index\", \"block_number\", \"block_timestamp\", \"address\", \"topics\", \"data\", \"raw_json\") VALUES ({}, '{}', 'raw_tx_{}', {}, {}, {}, 'address_{}', '[\"topic_{}\"]', 'data_{}', '{{\"event\":\"raw_{}\"}}');\n",
1265-
chain_id,
1266-
raindex,
1267-
label,
1268-
base,
1269-
block(base),
1270-
ts(base),
1271-
label,
1272-
label,
1273-
label,
1274-
label
1275-
));
1276-
12771269
out.push_str(&format!(
12781270
"INSERT INTO \"deposits\" (\"chain_id\", \"raindex_address\", \"transaction_hash\", \"log_index\", \"block_number\", \"block_timestamp\", \"sender\", \"token\", \"vault_id\", \"deposit_amount\", \"deposit_amount_uint256\") VALUES ({}, '{}', 'dep_tx_{}', {}, {}, {}, 'sender_{}', 'token_{}', 'vault_{}', 'amount_{}', 'uint_{}');\n",
12791271
chain_id,
@@ -1349,24 +1341,6 @@ mod tests {
13491341
label
13501342
));
13511343

1352-
out.push_str(&format!(
1353-
"INSERT INTO \"take_order_contexts\" (\"chain_id\", \"raindex_address\", \"transaction_hash\", \"log_index\", \"context_index\", \"context_value\") VALUES ({}, '{}', 'take_tx_{}', {}, 0, 'context_entry_{}');\n",
1354-
chain_id,
1355-
raindex,
1356-
label,
1357-
take_idx,
1358-
label
1359-
));
1360-
1361-
out.push_str(&format!(
1362-
"INSERT INTO \"context_values\" (\"chain_id\", \"raindex_address\", \"transaction_hash\", \"log_index\", \"context_index\", \"value_index\", \"value\") VALUES ({}, '{}', 'take_tx_{}', {}, 0, 0, 'context_value_{}');\n",
1363-
chain_id,
1364-
raindex,
1365-
label,
1366-
take_idx,
1367-
label
1368-
));
1369-
13701344
out.push_str(&format!(
13711345
"INSERT INTO \"clear_v3_events\" (\"chain_id\", \"raindex_address\", \"transaction_hash\", \"log_index\", \"block_number\", \"block_timestamp\", \"sender\", \"alice_order_hash\", \"alice_order_owner\", \"alice_input_io_index\", \"alice_output_io_index\", \"alice_bounty_vault_id\", \"alice_input_vault_id\", \"alice_output_vault_id\", \"bob_order_hash\", \"bob_order_owner\", \"bob_input_io_index\", \"bob_output_io_index\", \"bob_bounty_vault_id\", \"bob_input_vault_id\", \"bob_output_vault_id\") VALUES ({}, '{}', 'clear_tx_{}', {}, {}, {}, 'clear_sender_{}', 'alice_hash_{}', 'alice_owner_{}', 0, 1, 'alice_bounty_{}', 'alice_input_{}', 'alice_output_{}', 'bob_hash_{}', 'bob_owner_{}', 2, 3, 'bob_bounty_{}', 'bob_input_{}', 'bob_output_{}');\n",
13721346
chain_id,

0 commit comments

Comments
 (0)