Skip to content

Commit 1944ebe

Browse files
fix: table analysis memory (#570)
* fix(engine): keep decision table analysis memory and time in proportion to the table Schema queries no longer run table verification; graph table checks are deferred to diagnostics. Gap search and unreachable-rule cover run on per-dimension bitsets when the table is small enough to index densely, and on the set-based path otherwise. Missing cases are merged on interned ids and rendered lazily. Compression works on borrowed rows and interned cell ids, assigned once per round. Small string sets are held inline or as a sorted list instead of a tree. Findings are unchanged: the previous implementations stay as test-only references and are compared on generated tables, and tests/table_scaling.rs bounds peak memory with a counting allocator. * fix(nodejs): accept bytes and JSON text in Workspace setDocument, setPolicy and enhanceGraphTrace Documents with more than one million JSON values were rejected by the object conversion limit, which a two-column decision table reaches at 250,000 rows. Buffers and strings are parsed directly, and objects are converted without that limit. * ci: pin Rust to 1.99.0 Rust 1.100 passes one -L dependency directory per dependency, which pushes the rustc command for zen-engine past the Windows command-line limit when sccache spawns it (os error 206). Every workflow that installed stable now installs 1.99.0; miri stays on nightly. * ci: pin the UniFFI native build to Rust 1.99.0 actions-rust-cross installs its own toolchain and defaults to stable, so the Windows and macOS UniFFI builds were still on 1.100.
1 parent 9c374e1 commit 1944ebe

28 files changed

Lines changed: 4177 additions & 545 deletions

File tree

‎.github/workflows/benchmark.yaml‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@ jobs:
2020
steps:
2121
- uses: actions/checkout@v3
2222
- name: Install Rust
23-
run: rustup update stable
23+
run: |
24+
rustup toolchain install 1.99.0 --profile minimal
25+
rustup default 1.99.0
2426
- uses: ./.github/actions/setup-rust
2527
with:
2628
shared-key: benchmark-rust
@@ -48,9 +50,9 @@ jobs:
4850
cache: yarn
4951
cache-dependency-path: 'bindings/nodejs/yarn.lock'
5052
- name: Install Rust
51-
uses: dtolnay/rust-toolchain@stable
53+
uses: dtolnay/rust-toolchain@1.99.0
5254
with:
53-
toolchain: stable
55+
toolchain: 1.99.0
5456
targets: x86_64-unknown-linux-gnu
5557
- uses: ./.github/actions/setup-rust
5658
with:
@@ -75,7 +77,9 @@ jobs:
7577
with:
7678
python-version: '3.13'
7779
- name: Install Rust
78-
run: rustup update stable
80+
run: |
81+
rustup toolchain install 1.99.0 --profile minimal
82+
rustup default 1.99.0
7983
- uses: ./.github/actions/setup-rust
8084
with:
8185
shared-key: benchmark-python

‎.github/workflows/go.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ jobs:
4545

4646
- uses: actions-rs/toolchain@v1
4747
with:
48-
toolchain: stable
48+
toolchain: 1.99.0
4949
override: true
5050
target: ${{ matrix.settings.target }}
5151
- uses: ./.github/actions/setup-rust

‎.github/workflows/node.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -122,9 +122,9 @@ jobs:
122122
cache-dependency-path: 'bindings/nodejs/yarn.lock'
123123

124124
- name: Install
125-
uses: dtolnay/rust-toolchain@stable
125+
uses: dtolnay/rust-toolchain@1.99.0
126126
with:
127-
toolchain: stable
127+
toolchain: 1.99.0
128128
targets: ${{ matrix.settings.target }}
129129

130130
- name: Rust cache + sccache (+ zig for linux legs)

‎.github/workflows/python.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ jobs:
6262
args: ${{ matrix.args }}
6363
manylinux: ${{ matrix.manylinux || 'auto' }}
6464
sccache: 'true'
65+
rust-toolchain: 1.99.0
6566

6667
- name: Upload wheels
6768
uses: actions/upload-artifact@v4

‎.github/workflows/release-please.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ jobs:
7171
ref: ${{ steps.release.outputs.pr-branch }}
7272
token: ${{ secrets.PAT }}
7373

74-
- uses: dtolnay/rust-toolchain@stable
74+
- uses: dtolnay/rust-toolchain@1.99.0
7575
if: steps.release.outputs.pr-branch
7676

7777
- name: Update Cargo.lock on release PR

‎.github/workflows/rust.yaml‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,9 @@ jobs:
4141
steps:
4242
- uses: actions/checkout@v3
4343
- name: Install Rust
44-
run: rustup update stable
44+
run: |
45+
rustup toolchain install 1.99.0 --profile minimal
46+
rustup default 1.99.0
4547
- uses: ./.github/actions/setup-rust
4648
with:
4749
shared-key: rust-test-${{ matrix.os }}
@@ -58,7 +60,9 @@ jobs:
5860
steps:
5961
- uses: actions/checkout@v3
6062
- name: Install Rust
61-
run: rustup update stable
63+
run: |
64+
rustup toolchain install 1.99.0 --profile minimal --component rustfmt
65+
rustup default 1.99.0
6266
- run: cargo fmt --all -- --check
6367

6468
release:
@@ -73,7 +77,9 @@ jobs:
7377
- uses: actions/checkout@v3
7478

7579
- name: Install Rust
76-
run: rustup update stable
80+
run: |
81+
rustup toolchain install 1.99.0 --profile minimal
82+
rustup default 1.99.0
7783
7884
- name: Login to Crates.IO
7985
run: cargo login ${{ secrets.CRATES_TOKEN }}

‎.github/workflows/uniffi.yaml‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ jobs:
6868
- uses: actions/checkout@v3
6969

7070
- name: Setup Rust Toolchain
71-
uses: dtolnay/rust-toolchain@stable
71+
uses: dtolnay/rust-toolchain@1.99.0
7272
with:
7373
target: ${{ matrix.settings.target }}
7474

@@ -97,6 +97,7 @@ jobs:
9797
with:
9898
working-directory: ${{ env.UNIFFI_DIRECTORY }}
9999
target: ${{ matrix.settings.target }}
100+
toolchain: 1.99.0
100101
args: '--lib --release --locked ${{ matrix.settings.extra_args }}'
101102

102103
- name: Verify max glibc requirement
@@ -156,7 +157,7 @@ jobs:
156157
dotnet-version: '8.0.x'
157158

158159
- name: Setup Rust Toolchain
159-
uses: dtolnay/rust-toolchain@stable
160+
uses: dtolnay/rust-toolchain@1.99.0
160161

161162
- uses: ./.github/actions/setup-rust
162163
with:
@@ -234,7 +235,7 @@ jobs:
234235
- uses: actions/checkout@v3
235236

236237
- name: Setup Rust Toolchain
237-
uses: dtolnay/rust-toolchain@stable
238+
uses: dtolnay/rust-toolchain@1.99.0
238239

239240
- uses: ./.github/actions/setup-rust
240241
with:
@@ -287,7 +288,7 @@ jobs:
287288
- uses: actions/checkout@v3
288289

289290
- name: Setup Rust Toolchain
290-
uses: dtolnay/rust-toolchain@stable
291+
uses: dtolnay/rust-toolchain@1.99.0
291292

292293
- uses: ./.github/actions/setup-rust
293294
with:
@@ -332,7 +333,7 @@ jobs:
332333
distribution: 'temurin'
333334

334335
- name: Setup Rust Toolchain
335-
uses: dtolnay/rust-toolchain@stable
336+
uses: dtolnay/rust-toolchain@1.99.0
336337

337338
- uses: ./.github/actions/setup-rust
338339
with:
@@ -408,7 +409,7 @@ jobs:
408409
dotnet-version: '8.0.x'
409410

410411
- name: Setup Rust Toolchain
411-
uses: dtolnay/rust-toolchain@stable
412+
uses: dtolnay/rust-toolchain@1.99.0
412413

413414
- uses: ./.github/actions/setup-rust
414415
with:
@@ -501,7 +502,7 @@ jobs:
501502
- uses: actions/checkout@v3
502503

503504
- name: Setup Rust Toolchain
504-
uses: dtolnay/rust-toolchain@stable
505+
uses: dtolnay/rust-toolchain@1.99.0
505506
with:
506507
target: ${{ matrix.settings.target }}
507508

@@ -537,7 +538,7 @@ jobs:
537538
- uses: actions/checkout@v3
538539

539540
- name: Setup Rust Toolchain
540-
uses: dtolnay/rust-toolchain@stable
541+
uses: dtolnay/rust-toolchain@1.99.0
541542

542543
- uses: ./.github/actions/setup-rust
543544
with:
@@ -717,7 +718,7 @@ jobs:
717718
- uses: actions/checkout@v3
718719

719720
- name: Setup Rust Toolchain
720-
uses: dtolnay/rust-toolchain@stable
721+
uses: dtolnay/rust-toolchain@1.99.0
721722

722723
- uses: ./.github/actions/setup-rust
723724
with:

‎bindings/nodejs/src/policy.rs‎

Lines changed: 36 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,30 @@ use std::collections::HashMap;
22
use std::sync::Arc;
33

44
use napi::anyhow::anyhow;
5-
use napi::bindgen_prelude::{FnArgs, FunctionRef};
5+
use napi::bindgen_prelude::{Either3, FnArgs, FunctionRef, Object, Uint8Array};
66
use napi::Env;
77
use napi_derive::napi;
8+
use serde::de::DeserializeOwned;
89
use serde_json::Value;
910

1011
use zen_engine::policy::PolicyDocument;
1112
use zen_engine::workspace;
1213

1314
type ResolverRef = FunctionRef<FnArgs<(String, Value)>, Option<String>>;
1415

16+
type JsonInput<'env> = Either3<Uint8Array, String, Object<'env>>;
17+
18+
fn parse_json_input<T: DeserializeOwned>(env: &Env, input: JsonInput) -> Result<T, String> {
19+
match input {
20+
Either3::A(bytes) => serde_json::from_slice(bytes.as_ref()).map_err(|e| e.to_string()),
21+
Either3::B(text) => serde_json::from_str(&text).map_err(|e| e.to_string()),
22+
Either3::C(object) => {
23+
let value: Value = env.from_js_value(object).map_err(|e| e.to_string())?;
24+
serde_json::from_value(value).map_err(|e| e.to_string())
25+
}
26+
}
27+
}
28+
1529
#[napi(object)]
1630
pub struct PolicyChanges {
1731
pub cursor: i64,
@@ -593,16 +607,26 @@ impl Workspace {
593607
}
594608

595609
#[napi]
596-
pub fn set_document(&mut self, path: String, document: Value) -> napi::Result<()> {
610+
pub fn set_document(
611+
&mut self,
612+
env: Env,
613+
path: String,
614+
#[napi(ts_arg_type = "Uint8Array | string | object")] document: JsonInput,
615+
) -> napi::Result<()> {
597616
let doc: zen_engine::model::DecisionContent =
598-
serde_json::from_value(document).map_err(|e| anyhow!("Invalid document: {e}"))?;
617+
parse_json_input(&env, document).map_err(|e| anyhow!("Invalid document: {e}"))?;
599618
self.inner.set_document(path, doc);
600619
Ok(())
601620
}
602621

603622
#[napi]
604-
pub fn set_policy(&mut self, path: String, document: Value) -> napi::Result<()> {
605-
let doc: PolicyDocument = serde_json::from_value(document)
623+
pub fn set_policy(
624+
&mut self,
625+
env: Env,
626+
path: String,
627+
#[napi(ts_arg_type = "Uint8Array | string | object")] document: JsonInput,
628+
) -> napi::Result<()> {
629+
let doc: PolicyDocument = parse_json_input(&env, document)
606630
.map_err(|e| anyhow!("Invalid policy document: {e}"))?;
607631
self.inner.set_policy(path, doc);
608632
Ok(())
@@ -981,9 +1005,14 @@ impl Workspace {
9811005
}
9821006

9831007
#[napi(ts_return_type = "PolicyTrace")]
984-
pub fn enhance_graph_trace(&self, path: String, trace: Value) -> napi::Result<Value> {
1008+
pub fn enhance_graph_trace(
1009+
&self,
1010+
env: Env,
1011+
path: String,
1012+
#[napi(ts_arg_type = "Uint8Array | string | object")] trace: JsonInput,
1013+
) -> napi::Result<Value> {
9851014
let trace_map: workspace::GraphTraceMap =
986-
serde_json::from_value(trace).map_err(|e| napi::Error::from_reason(e.to_string()))?;
1015+
parse_json_input(&env, trace).map_err(napi::Error::from_reason)?;
9871016
let result = self
9881017
.inner
9891018
.enhance_graph_trace(&Arc::from(path.as_str()), &trace_map)
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
import { describe, expect, it } from '@jest/globals';
2+
3+
const { Workspace } = require('../index.js');
4+
5+
const lookupTable = (rows: number) => ({
6+
nodes: [
7+
{ id: 'in', name: 'request', type: 'inputNode', content: { schema: '' }, position: { x: 0, y: 0 } },
8+
{
9+
id: 'table',
10+
name: 'lookup',
11+
type: 'decisionTableNode',
12+
position: { x: 300, y: 0 },
13+
content: {
14+
hitPolicy: 'first',
15+
inputs: [{ id: 'code', name: 'Code', field: 'code' }],
16+
outputs: [{ id: 'rate', name: 'Rate', field: 'rate' }],
17+
rules: Array.from({ length: rows }, (_, row) => ({
18+
_id: `r${row}`,
19+
code: `"C${row}"`,
20+
rate: String(row),
21+
})),
22+
passThrough: true,
23+
},
24+
},
25+
{ id: 'out', name: 'response', type: 'outputNode', content: { schema: '' }, position: { x: 600, y: 0 } },
26+
],
27+
edges: [
28+
{ id: 'e1', type: 'edge', sourceId: 'in', targetId: 'table' },
29+
{ id: 'e2', type: 'edge', sourceId: 'table', targetId: 'out' },
30+
],
31+
});
32+
33+
describe('Workspace large documents', () => {
34+
const rows = 300_000;
35+
const content = lookupTable(rows);
36+
const text = JSON.stringify(content);
37+
38+
it.each<[string, () => unknown]>([
39+
['an object', () => content],
40+
['a JSON string', () => text],
41+
['a Buffer', () => Buffer.from(text)],
42+
['a Uint8Array', () => new TextEncoder().encode(text)],
43+
])('loads more than a million JSON values from %s', (_, document) => {
44+
const ws = new Workspace();
45+
ws.setDocument('lookup', document());
46+
expect(ws.isGraph('lookup')).toBe(true);
47+
expect(ws.paths()).toEqual(['lookup']);
48+
expect(ws.inputs({ policyPath: 'lookup' }).map((input: any) => input.path)).toEqual(['code']);
49+
});
50+
51+
it('reports invalid JSON text', () => {
52+
const ws = new Workspace();
53+
expect(() => ws.setDocument('broken', '{ "nodes": ')).toThrow(/Invalid document/);
54+
});
55+
});

0 commit comments

Comments
 (0)