Skip to content

Commit b764040

Browse files
authored
fix(engine): bound table gap analysis memory, analyse expression columns (#549)
* fix(engine): bound table gap analysis memory, analyse expression columns Missing-case detection built the full atom/row partition for every column before charging the work budget. Nested ranges (`> 1`, `> 2`, ...) make that partition quadratic, so large tables (tens of thousands of rows) allocated gigabytes before giving up, exhausting the 1GB WASM memory. The partition size is now counted first and the budget checked before building it; results are unchanged (50k rows: ~10GB/28s -> ~650MB/1s natively). Columns without a field whose filled cells are all conditions on one field (`age > 50`, `age > 30`) are now analysed like field columns on that field: duplicates, unreachable rows, missing cases and compression. Compressed and missing-case cells are written back as full expressions (`age > 30`). Columns mixing fields or using non-analysable expressions stay opaque. * chore(engine): remove comments from table analysis changes
1 parent c8a7d89 commit b764040

8 files changed

Lines changed: 466 additions & 18 deletions

File tree

‎core/engine/src/analysis/table/merge.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ impl VerifyTable<'_> {
8686
.map(|mut row| {
8787
for &col in &row.stale {
8888
if let CellConstraint::Known(set) = &row.cells[col] {
89-
if let Some(text) = CellText::of(set, &ValueSet::all(), false) {
89+
if let Some(text) = self.cell_text(col, set, &ValueSet::all(), false) {
9090
row.rule
9191
.insert(self.inputs[col].id.clone(), Arc::from(text.as_str()));
9292
}
@@ -366,7 +366,8 @@ impl VerifyTable<'_> {
366366
return false;
367367
}
368368
let union = a.union(&b);
369-
let Some(text) = CellText::of(&union, &ValueSet::all(), self.inputs[col].dated) else {
369+
let Some(text) = self.cell_text(col, &union, &ValueSet::all(), self.inputs[col].dated)
370+
else {
370371
return false;
371372
};
372373
let id = self.inputs[col].id.clone();

‎core/engine/src/analysis/table/missing.rs‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -258,9 +258,10 @@ impl VerifyTable<'_> {
258258
if !dim.input {
259259
example = None;
260260
}
261-
if let Some(cells) = cells.as_mut() {
262-
if let Some(&first) = dim.columns.first() {
263-
cells.push((self.inputs[first].id.clone(), text.clone()));
261+
if let (Some(list), Some(&first)) = (cells.as_mut(), dim.columns.first()) {
262+
match self.cell_text(first, set, &dim.domain, dim.dated) {
263+
Some(cell) => list.push((self.inputs[first].id.clone(), cell)),
264+
None => cells = None,
264265
}
265266
}
266267
parts.push((dim.label.clone(), text));
@@ -389,7 +390,11 @@ impl Gaps<'_> {
389390
if sets.iter().all(|set| set == bounds) {
390391
continue;
391392
}
392-
let groups = Partition::groups(bounds, &sets);
393+
let partition = Partition::new(bounds, &sets);
394+
if partition.touches(&sets) > self.work {
395+
return None;
396+
}
397+
let groups = partition.groups(&sets);
393398
if groups.len() < 2 {
394399
continue;
395400
}

‎core/engine/src/analysis/table/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ impl TableColumn {
7878
prefer: resolved.and_then(Self::kind_of),
7979
label,
8080
domain: field_type.and_then(Self::domain_of),
81+
subject: None,
8182
}
8283
}
8384

‎core/engine/src/analysis/table/partition.rs‎

Lines changed: 75 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -94,8 +94,49 @@ pub(super) struct Partition {
9494
}
9595

9696
impl Partition {
97-
pub(super) fn groups(region: &ValueSet, sets: &[ValueSet]) -> Vec<(ValueSet, Vec<usize>)> {
98-
let partition = Self::new(region, sets);
97+
pub(super) fn touches(&self, sets: &[ValueSet]) -> usize {
98+
let mut present = Vec::with_capacity(self.pieces.len() + 1);
99+
present.push(0usize);
100+
for piece in &self.pieces {
101+
present.push(present[present.len() - 1] + usize::from(piece.is_some()));
102+
}
103+
sets.iter()
104+
.map(|set| {
105+
let strings = match &set.strings {
106+
StringSet::Finite(keys) => keys
107+
.iter()
108+
.filter(|k| self.strings.contains_key(*k))
109+
.count(),
110+
StringSet::CoFinite(excluded) => {
111+
let listed_excluded = excluded
112+
.iter()
113+
.filter(|k| self.strings.contains_key(*k))
114+
.count();
115+
self.strings.len() - listed_excluded
116+
+ usize::from(self.string_rest.is_some())
117+
}
118+
};
119+
let numbers: usize = set
120+
.numbers
121+
.intervals()
122+
.iter()
123+
.filter_map(|interval| self.points.range(interval))
124+
.map(|(first, last)| present[last + 1] - present[first])
125+
.sum();
126+
let bools = [ValueSet::FALSE, ValueSet::TRUE]
127+
.into_iter()
128+
.enumerate()
129+
.filter(|(bit, slot)| set.bools & slot != 0 && self.bools[*bit].is_some())
130+
.count();
131+
let null = usize::from(set.null && self.null.is_some());
132+
let other = usize::from(set.other && self.other.is_some());
133+
strings + numbers + bools + null + other
134+
})
135+
.sum()
136+
}
137+
138+
pub(super) fn groups(&self, sets: &[ValueSet]) -> Vec<(ValueSet, Vec<usize>)> {
139+
let partition = self;
99140
let mut touched: Vec<Vec<usize>> = vec![Vec::new(); partition.atoms.len()];
100141
for (id, set) in sets.iter().enumerate() {
101142
for atom in partition.touching(set) {
@@ -120,7 +161,7 @@ impl Partition {
120161
.collect()
121162
}
122163

123-
fn new(region: &ValueSet, sets: &[ValueSet]) -> Self {
164+
pub(super) fn new(region: &ValueSet, sets: &[ValueSet]) -> Self {
124165
let mut partition = Self {
125166
atoms: Vec::new(),
126167
strings: HashMap::default(),
@@ -246,3 +287,34 @@ impl Partition {
246287
out
247288
}
248289
}
290+
291+
#[cfg(test)]
292+
mod tests {
293+
use super::*;
294+
295+
fn above(n: i64) -> ValueSet {
296+
ValueSet::numbers(NumberSet::from_intervals(vec![Interval::new(
297+
Bound::Exclusive(Decimal::from(n)),
298+
Bound::Unbounded,
299+
)]))
300+
}
301+
302+
#[test]
303+
fn touches_counts_what_groups_list() {
304+
let mut sets: Vec<ValueSet> = (0..40).map(above).collect();
305+
sets.extend((0..10).map(|i| ValueSet::string(&format!("s{i}"))));
306+
sets.push(ValueSet::string("s1").complement());
307+
sets.push(ValueSet::bool(true).union(&ValueSet::null()));
308+
sets.push(above(5).union(&ValueSet::string("s2")));
309+
let region = ValueSet::all();
310+
let partition = Partition::new(&region, &sets);
311+
let listed: usize = sets.iter().map(|set| partition.touching(set).len()).sum();
312+
assert_eq!(partition.touches(&sets), listed);
313+
let grouped: usize = partition
314+
.groups(&sets)
315+
.iter()
316+
.map(|(_, ids)| ids.len())
317+
.sum();
318+
assert!(grouped <= partition.touches(&sets));
319+
}
320+
}

‎core/engine/src/analysis/table/verify.rs‎

Lines changed: 84 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ use zen_expression::intellisense::IntelliSense;
77

88
use super::cell::CellConstraint;
99
use super::index::RowIndex;
10-
use super::print::DateDay;
10+
use super::print::{CellText, DateDay};
1111
use super::value_set::{ValueKind, ValueSet};
1212

1313
pub(crate) const MAX_INPUTS: usize = 30;
@@ -35,6 +35,7 @@ pub(crate) struct VerifyInput {
3535
pub(crate) prefer: Option<ValueKind>,
3636
pub(crate) label: Arc<str>,
3737
pub(crate) domain: Option<ValueSet>,
38+
pub(crate) subject: Option<Vec<Rc<str>>>,
3839
}
3940

4041
#[derive(Debug, Clone)]
@@ -116,8 +117,77 @@ impl Budget {
116117
}
117118
}
118119

119-
impl VerifyTable<'_> {
120+
impl<'a> VerifyTable<'a> {
120121
pub(crate) fn verify(&self, is: &mut IntelliSense) -> Vec<Finding> {
122+
match self.with_expression_subjects(is) {
123+
Some(resolved) => resolved.verify_resolved(is),
124+
None => self.verify_resolved(is),
125+
}
126+
}
127+
128+
fn with_expression_subjects(&self, is: &mut IntelliSense) -> Option<VerifyTable<'a>> {
129+
let mut inputs = self.inputs.clone();
130+
let mut changed = false;
131+
for input in inputs.iter_mut().filter(|input| input.field.is_none()) {
132+
let mut subject: Option<Vec<Rc<str>>> = None;
133+
let mut consistent = true;
134+
for rule in self.rules {
135+
let source = rule.get(&input.id).map(|c| c.as_ref()).unwrap_or("");
136+
if source.trim().is_empty() {
137+
continue;
138+
}
139+
match (CellConstraint::subject_of(is, source), &subject) {
140+
(Some(found), None) => subject = Some(found),
141+
(Some(found), Some(known)) if found == *known => {}
142+
_ => {
143+
consistent = false;
144+
break;
145+
}
146+
}
147+
}
148+
let (true, Some(subject)) = (consistent, subject) else {
149+
continue;
150+
};
151+
let field: Arc<str> = Arc::from(subject.join("."));
152+
let plain = subject.iter().all(|segment| {
153+
let mut chars = segment.chars();
154+
chars
155+
.next()
156+
.is_some_and(|c| c.is_ascii_alphabetic() || c == '_')
157+
&& chars.all(|c| c.is_ascii_alphanumeric() || c == '_')
158+
});
159+
input.analyzable = true;
160+
input.path = plain.then(|| field.clone());
161+
if input.label.trim().is_empty() {
162+
input.label = field.clone();
163+
}
164+
input.field = Some(field);
165+
input.subject = Some(subject);
166+
changed = true;
167+
}
168+
changed.then(|| VerifyTable {
169+
mode: self.mode,
170+
inputs,
171+
outputs: self.outputs.clone(),
172+
rules: self.rules,
173+
fallible: self.fallible.clone(),
174+
})
175+
}
176+
177+
pub(super) fn cell_text(
178+
&self,
179+
col: usize,
180+
set: &ValueSet,
181+
domain: &ValueSet,
182+
dated: bool,
183+
) -> Option<String> {
184+
match &self.inputs[col].subject {
185+
Some(subject) => CellText::standard(set, domain, dated, &subject.join(".")),
186+
None => CellText::of(set, domain, dated),
187+
}
188+
}
189+
190+
fn verify_resolved(&self, is: &mut IntelliSense) -> Vec<Finding> {
121191
let mut findings = Vec::new();
122192
if self.inputs.is_empty() {
123193
return findings;
@@ -130,7 +200,18 @@ impl VerifyTable<'_> {
130200
.iter()
131201
.map(|col| {
132202
let source = rule.get(&col.id).map(|c| c.as_ref()).unwrap_or("");
133-
CellConstraint::parse(is, source, col.unary, col.analyzable, col.dated)
203+
match &col.subject {
204+
Some(subject) => {
205+
CellConstraint::parse_subject(is, source, subject, col.dated)
206+
}
207+
None => CellConstraint::parse(
208+
is,
209+
source,
210+
col.unary,
211+
col.analyzable,
212+
col.dated,
213+
),
214+
}
134215
})
135216
.collect()
136217
})

‎core/engine/tests/table_verification.rs‎

Lines changed: 80 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -686,12 +686,17 @@ fn opaque_cells_never_create_gaps() {
686686
}
687687
.assert_gaps(
688688
Severity::Hint,
689-
"no row matches 1 input case: In 1 false",
690-
json!([{
691-
"cells": { "i1": "false" },
692-
"description": "In 1 false",
693-
"example": { "applicant": { "vip": false } }
694-
}]),
689+
"no row matches 2 input cases: In 0 <= 10; In 0 > 10 and In 1 false",
690+
json!([
691+
{
692+
"cells": { "i0": "applicant.age <= 10" },
693+
"description": "In 0 <= 10"
694+
},
695+
{
696+
"cells": { "i0": "applicant.age > 10", "i1": "false" },
697+
"description": "In 0 > 10 and In 1 false"
698+
}
699+
]),
695700
);
696701
}
697702

@@ -1831,3 +1836,72 @@ async fn collect_compression_keeps_the_order_of_results() {
18311836
assert_eq!(before, after, "{input}");
18321837
}
18331838
}
1839+
1840+
#[test]
1841+
fn expression_columns_on_one_field_compress_like_field_columns() {
1842+
Table {
1843+
hit: "first",
1844+
inputs: &[""],
1845+
outputs: &["applicant.discount"],
1846+
rows: &[
1847+
("r1", &["applicant.age > 50"], &["true"]),
1848+
("r2", &["applicant.age > 30"], &["true"]),
1849+
],
1850+
}
1851+
.assert_compressed(Some((2, &["i0=applicant.age > 30 o0=true"])));
1852+
}
1853+
1854+
#[test]
1855+
fn expression_columns_on_one_field_report_rows_that_never_fire() {
1856+
let table = Table {
1857+
hit: "first",
1858+
inputs: &[""],
1859+
outputs: &["applicant.discount"],
1860+
rows: &[
1861+
("r1", &["applicant.age > 30"], &["1"]),
1862+
("r2", &["applicant.age > 50 and applicant.age < 60"], &["2"]),
1863+
],
1864+
};
1865+
let policy = table.policy_findings();
1866+
assert_eq!(policy, table.graph_findings());
1867+
assert_eq!(policy.len(), 1, "{policy:?}");
1868+
assert!(policy[0].contains("r2"), "{policy:?}");
1869+
}
1870+
1871+
#[test]
1872+
fn expression_columns_write_missing_cases_as_expressions() {
1873+
let table = Table {
1874+
hit: "first",
1875+
inputs: &[""],
1876+
outputs: &["applicant.discount"],
1877+
rows: &[("r1", &["applicant.age > 30"], &["1"])],
1878+
};
1879+
for (kind, diagnostics) in [
1880+
("policy", table.policy_diagnostics()),
1881+
("graph", table.graph_diagnostics()),
1882+
] {
1883+
let gaps = Table::gaps(diagnostics).unwrap_or_else(|| panic!("{kind}: no MissingCases"));
1884+
assert_eq!(
1885+
gaps.cases[0]["cells"]["i0"],
1886+
json!("applicant.age <= 30"),
1887+
"{kind}"
1888+
);
1889+
}
1890+
}
1891+
1892+
#[test]
1893+
fn expression_columns_mixing_fields_stay_opaque() {
1894+
let table = Table {
1895+
hit: "first",
1896+
inputs: &[""],
1897+
outputs: &["applicant.discount"],
1898+
rows: &[
1899+
("r1", &["applicant.age > 30"], &["1"]),
1900+
("r2", &["applicant.tier == \"gold\""], &["1"]),
1901+
("r3", &["applicant.age > 50"], &["1"]),
1902+
],
1903+
};
1904+
table.assert_compressed(None);
1905+
assert!(table.policy_findings().is_empty());
1906+
assert!(table.graph_findings().is_empty());
1907+
}

0 commit comments

Comments
 (0)