diff --git a/docs/value/set.md b/docs/value/set.md new file mode 100644 index 00000000..6a78c363 --- /dev/null +++ b/docs/value/set.md @@ -0,0 +1,79 @@ +# Set + +Opaque container for `Value::Set`'s element storage, enabling alternative +backends without call-site changes. Pairs with [`Object`](object.md) under +a shared design philosophy. + +## Design + +`Set` wraps a `BTreeSet` today but exposes only a curated method +surface (`contains`, `insert`, `remove`, `iter`, `iter_sorted`, `cursor`, +`is_subset`, `intersection`, `union`, `difference`, serde). The inner set is +private — callers cannot pattern-match it or hand out references to the +backing store, so the backend can change without churn at the ~400 call +sites that name `Set`. + +Two iteration methods reflect a real distinction: `iter()` makes no +ordering promise (lets future hash/lazy backends skip sorting work); +`iter_sorted()` guarantees deterministic order (used by serialization and +`Ord`). Cursor types support incremental traversal needed by the RVM +iteration state without exposing iterator internals. + +`Ord` is hand-written against `iter_sorted` rather than derived, so two +backends that store elements differently still compare equal when their +sorted contents match. + +## Scenarios enabled + +- **Hash-backed storage** — `FxHashSet`-backed inner turns O(log n) + membership checks into O(1); swap in for policies where elements aren't + compared ordinally. +- **Lazy/streaming** — wrap a `LazySetProvider` (DB query, CBOR slice, + REST endpoint) and materialize elements on demand. +- **Arena allocation** — bumpalo-backed inner for eval-time temporaries; + drop the whole arena at query end with zero per-element free cost. +- **FFI-backed** — host-language collections (Python set, JS Set) without + copying into Rust. +- **Bloom-filter pre-check** — front a large backing set with a Bloom + filter for fast negative-membership tests on read-mostly allowlists. + +## Known use cases + +- **Azure Policy allowed-values lists** — large allowlists (allowed + regions, allowed SKUs, allowed image publishers) compared against + single resource values. Hash-backed Set turns O(log n) membership + checks into O(1). +- **SARIF rule deduplication** — collapsing duplicate rule references + across thousands of result records. Set-of-objects with structural + hashing avoids the BTreeSet sort cost on every insert. +- **RBAC role membership** — checking whether a principal belongs to any + of dozens of role groups. Hash-backed Set scales to thousands of + members with constant-time membership. +- **Azure Policy denied-resource-type sets** — exclusion lists used by + deny-effect policies; same hash-backed pattern as allowed-values. + +## Precedents + +- **`indexmap::IndexSet`** — opaque newtype that pairs hash lookup with + insertion-order iteration; precedent for "Set with alternative + ordering semantics behind a stable surface." +- **`hashbrown::HashSet`** — backs Rust's `std::collections::HashSet` + and demonstrates a fully swappable backend behind a stable API. +- **`roaring::RoaringBitmap`** — bitmap-backed integer set. Not + applicable to `Value` keys directly, but a precedent for the broader + idea of "Set with alternative storage representations chosen by + workload shape." +- **`serde_json`** — note that `serde_json` has no Set equivalent: its + Value enum collapses sets into arrays. Regorus's first-class Set with + storage abstraction is therefore unusually well-positioned among JSON + value libraries. + +## Notes + +Cursor types are `pub` (referenced by public `IterationState`) but not +re-exported at the crate root. The crate-internal `Set`/`Map`/`MapEntry` +aliases for `BTreeSet`/`BTreeMap` in `lib.rs` were renamed to +`MapSet`/`Map`/`MapEntry` when this type landed, to free the `Set` name +for the new public type. Future Array and String abstractions follow the +same shape — see `docs/value/array.md` and `docs/value/string.md` when +they land. diff --git a/src/compiled_policy.rs b/src/compiled_policy.rs index 22920323..edaa4d35 100644 --- a/src/compiled_policy.rs +++ b/src/compiled_policy.rs @@ -217,7 +217,7 @@ pub(crate) struct CompiledPolicyData { pub(crate) default_rules: Map>, pub(crate) imports: BTreeMap>, pub(crate) functions: FunctionTable, - pub(crate) rule_paths: Set, + pub(crate) rule_paths: MapSet, #[cfg(feature = "azure_policy")] pub(crate) target_info: Option, #[cfg(feature = "azure_policy")] diff --git a/src/lib.rs b/src/lib.rs index 00815809..953d8bbe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -205,10 +205,10 @@ pub use alloc::sync::Arc as Rc; pub use alloc::rc::Rc; #[cfg(feature = "std")] -use std::collections::{hash_map::Entry as MapEntry, HashMap as Map, HashSet as Set}; +use std::collections::{hash_map::Entry as MapEntry, HashMap as Map, HashSet as MapSet}; #[cfg(not(feature = "std"))] -use alloc::collections::{btree_map::Entry as MapEntry, BTreeMap as Map, BTreeSet as Set}; +use alloc::collections::{btree_map::Entry as MapEntry, BTreeMap as Map, BTreeSet as MapSet}; use alloc::{ borrow::ToOwned as _, diff --git a/src/value/mod.rs b/src/value/mod.rs index 7886ddcb..ca8607b1 100644 --- a/src/value/mod.rs +++ b/src/value/mod.rs @@ -12,16 +12,22 @@ )] // value helpers index paths directly for performance mod object; +mod set; #[cfg(test)] mod tests; #[allow(unused_imports)] // surface for downstream PRs pub use object::{IntoIter, Iter, IterMut, Object}; +#[allow(unused_imports)] // surface for downstream PRs +pub use set::Set; #[cfg(feature = "rvm")] #[allow(unused_imports)] // surface for downstream PRs pub use object::ObjectCursor; +#[cfg(feature = "rvm")] +#[allow(unused_imports)] // surface for downstream PRs +pub use set::SetCursor; use crate::number::Number; diff --git a/src/value/set/iter.rs b/src/value/set/iter.rs new file mode 100644 index 00000000..ffd1cb48 --- /dev/null +++ b/src/value/set/iter.rs @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Opaque iterator types for [`Set`]. +//! +//! These newtypes wrap the storage backend's iterators so the backend can be +//! swapped without changing any iterator type signatures observed by callers. + +use alloc::collections::btree_set; +use core::iter::FusedIterator; + +use super::Set; +use crate::value::Value; + +/// Owned iterator over `Value` elements. +#[derive(Debug)] +pub struct IntoIter { + pub(super) inner: btree_set::IntoIter, +} + +impl Iterator for IntoIter { + type Item = Value; + #[inline] + fn next(&mut self) -> Option { + self.inner.next() + } + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +impl DoubleEndedIterator for IntoIter { + #[inline] + fn next_back(&mut self) -> Option { + self.inner.next_back() + } +} + +impl ExactSizeIterator for IntoIter { + #[inline] + fn len(&self) -> usize { + self.inner.len() + } +} + +impl FusedIterator for IntoIter {} + +/// Borrowed iterator over `&Value` elements. +#[derive(Debug, Clone)] +pub struct Iter<'a> { + pub(super) inner: btree_set::Iter<'a, Value>, +} + +impl<'a> Iterator for Iter<'a> { + type Item = &'a Value; + #[inline] + fn next(&mut self) -> Option { + self.inner.next() + } + #[inline] + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +impl<'a> DoubleEndedIterator for Iter<'a> { + #[inline] + fn next_back(&mut self) -> Option { + self.inner.next_back() + } +} + +impl<'a> ExactSizeIterator for Iter<'a> { + #[inline] + fn len(&self) -> usize { + self.inner.len() + } +} + +impl<'a> FusedIterator for Iter<'a> {} + +impl IntoIterator for Set { + type Item = Value; + type IntoIter = IntoIter; + #[inline] + fn into_iter(self) -> Self::IntoIter { + IntoIter { + inner: self.inner.into_iter(), + } + } +} + +impl<'a> IntoIterator for &'a Set { + type Item = &'a Value; + type IntoIter = Iter<'a>; + #[inline] + fn into_iter(self) -> Self::IntoIter { + Iter { + inner: self.inner.iter(), + } + } +} diff --git a/src/value/set/mod.rs b/src/value/set/mod.rs new file mode 100644 index 00000000..d57fba9e --- /dev/null +++ b/src/value/set/mod.rs @@ -0,0 +1,269 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! See [`Set`]. + +mod iter; +mod serde; + +use alloc::collections::BTreeSet; +use core::cmp::Ordering; +use core::fmt; +use core::ops::Bound; + +use crate::value::Value; + +#[allow(unused_imports)] // surface for downstream PRs +pub use iter::{IntoIter, Iter}; + +/// Opaque, ordered set of [`Value`]s. +/// +/// The current backing storage is `BTreeSet`. The inner field is +/// private so the representation can change (hash-backed, lazy, bloom-fronted, +/// FFI-backed) without touching call sites. +/// +/// # Iteration +/// +/// - [`Set::iter`] — implementation-defined order; non-resumable. +/// - [`Set::iter_sorted`] — sorted by `Value::Ord`; non-resumable. +/// - [`Set::cursor`] / [`Set::next`] — implementation-defined order, +/// resumable; cheapest per-step cost. Used by interpreter/RVM when iteration +/// must yield mid-flight. +#[derive(Default, Clone, Eq, PartialEq)] +pub struct Set { + inner: BTreeSet, +} + +impl Set { + /// Create an empty `Set`. + #[inline] + pub const fn new() -> Self { + Self { + inner: BTreeSet::new(), + } + } + + #[inline] + pub fn len(&self) -> usize { + self.inner.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + #[inline] + pub fn contains(&self, value: &Value) -> bool { + self.inner.contains(value) + } + + #[inline] + pub fn get(&self, value: &Value) -> Option<&Value> { + self.inner.get(value) + } + + /// First element in sorted order (by `Value::Ord`). + #[inline] + pub fn first(&self) -> Option<&Value> { + self.iter_sorted().next() + } + + /// Last element in sorted order (by `Value::Ord`). + #[inline] + pub fn last(&self) -> Option<&Value> { + self.iter_sorted().next_back() + } + + /// Iteration in implementation-defined order. Non-resumable. + /// + /// For the current BTree-backed storage this happens to be sorted, but + /// callers MUST NOT depend on that. Use [`Set::iter_sorted`] when + /// deterministic order is required, or [`Set::cursor`] when iteration + /// must yield and resume. + #[inline] + pub fn iter(&self) -> impl Iterator + '_ { + self.inner.iter() + } + + /// Iteration in sorted order (by `Value::Ord`). Non-resumable. + /// + /// Use this for serialization, snapshots, hashing, `Debug`, etc. + #[inline] + pub fn iter_sorted(&self) -> Iter<'_> { + // BTree backend iterates sorted natively. + Iter { + inner: self.inner.iter(), + } + } + + /// Insert `value`. Returns `true` if the value was newly inserted. + #[inline] + pub fn insert(&mut self, value: Value) -> bool { + self.inner.insert(value) + } + + #[inline] + pub fn remove(&mut self, value: &Value) -> bool { + self.inner.remove(value) + } + + #[inline] + pub fn retain(&mut self, f: F) + where + F: FnMut(&Value) -> bool, + { + self.inner.retain(f); + } + + #[inline] + pub fn clear(&mut self) { + self.inner.clear(); + } + + #[inline] + pub fn append(&mut self, other: &mut Set) { + self.inner.append(&mut other.inner); + } + + /// Set intersection. Returns a new `Set` containing the elements + /// present in both `self` and `other`. + pub fn intersection(&self, other: &Set) -> Set { + Set { + inner: self.inner.intersection(&other.inner).cloned().collect(), + } + } + + /// Set union. Returns a new `Set` containing the elements present in + /// either `self` or `other`. + pub fn union(&self, other: &Set) -> Set { + Set { + inner: self.inner.union(&other.inner).cloned().collect(), + } + } + + /// Set difference. Returns a new `Set` containing the elements present + /// in `self` but not in `other`. + pub fn difference(&self, other: &Set) -> Set { + Set { + inner: self.inner.difference(&other.inner).cloned().collect(), + } + } + + #[inline] + pub fn is_subset(&self, other: &Set) -> bool { + self.inner.is_subset(&other.inner) + } + + /// Wrap into a `Value::Set`. + #[inline] + pub fn into_value(self) -> Value { + Value::Set(crate::Rc::new(self.inner)) + } + + /// Create a resumable cursor over elements in implementation-defined + /// order. Stable for the lifetime of `&self`. O(1). + /// + /// The cursor is fully self-owned (it stores a clone of the last-seen + /// element, not a reference) so it can be stored as a field of a + /// long-lived state struct — e.g. an RVM iteration frame that persists + /// across instruction dispatches. As a consequence, mutating the `Set` + /// between `next()` calls is not rejected by the borrow checker; the + /// resulting iteration order in that case is unspecified. + #[inline] + pub const fn cursor(&self) -> SetCursor { + SetCursor { + inner: SetCursorInner::BTree(None), + } + } + + /// Advance `cursor` and yield the next element. O(log n) for the BTree + /// backend (range probe); future hash/inline variants may be O(1). + pub fn next<'a>(&'a self, cursor: &mut SetCursor) -> Option<&'a Value> { + let SetCursorInner::BTree(ref mut last) = cursor.inner; + let next = last.as_ref().map_or_else( + || self.inner.iter().next(), + |prev| { + // `(Bound<&T>, Bound<&T>)` impls `RangeBounds` — no clone + // needed to build the resume bound. + self.inner + .range((Bound::Excluded(prev), Bound::Unbounded)) + .next() + }, + ); + let v = next?; + *last = Some(v.clone()); + Some(v) + } +} + +/// Opaque resumable cursor over a [`Set`]'s elements in +/// implementation-defined order. +/// +/// Self-owned: holds no borrow on the `Set`, so it can be stored as a +/// field of a long-lived state struct (e.g. an RVM iteration frame). +#[derive(Debug, Clone)] +pub struct SetCursor { + inner: SetCursorInner, +} + +#[derive(Debug, Clone)] +enum SetCursorInner { + /// BTree backend cursor: tracks last-seen element. `None` means "before start". + BTree(Option), +} + +// ---- Hand-written Ord/PartialOrd ---------------------------------------- +// +// Implemented in terms of `iter_sorted()` so ordering is consistent with the +// canonical (sorted) view of the elements and is therefore independent of +// the storage variant. + +impl Ord for Set { + fn cmp(&self, other: &Self) -> Ordering { + self.iter_sorted().cmp(other.iter_sorted()) + } +} + +impl PartialOrd for Set { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl fmt::Debug for Set { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Use sorted iteration so Debug output is stable across storage + // variants. + f.debug_set().entries(self.iter_sorted()).finish() + } +} + +impl Extend for Set { + fn extend>(&mut self, iter: I) { + self.inner.extend(iter); + } +} + +impl FromIterator for Set { + fn from_iter>(iter: I) -> Self { + Self { + inner: BTreeSet::from_iter(iter), + } + } +} + +impl From> for Set { + #[inline] + fn from(set: BTreeSet) -> Self { + Self { inner: set } + } +} + +impl From for Value { + #[inline] + fn from(s: Set) -> Self { + s.into_value() + } +} diff --git a/src/value/set/serde.rs b/src/value/set/serde.rs new file mode 100644 index 00000000..7816ca78 --- /dev/null +++ b/src/value/set/serde.rs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Serde `Serialize`/`Deserialize` impls for [`Set`]. + +use core::fmt; + +use serde::de::{Deserialize, Deserializer, Error as _, SeqAccess, Visitor}; +use serde::ser::{Serialize, Serializer}; + +use super::Set; +use crate::value::Value; + +impl Serialize for Set { + fn serialize(&self, serializer: S) -> Result { + // Sets serialize as JSON arrays. Sorted iteration: canonical output. + serializer.collect_seq(self.iter_sorted()) + } +} + +struct SetVisitor; + +impl<'de> Visitor<'de> for SetVisitor { + type Value = Set; + + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("a sequence of Values") + } + + fn visit_seq>(self, mut access: A) -> Result { + let mut set = Set::new(); + while let Some(v) = access.next_element::()? { + set.insert(v); + crate::utils::limits::check_memory_limit_if_needed().map_err(A::Error::custom)?; + } + Ok(set) + } +} + +impl<'de> Deserialize<'de> for Set { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_seq(SetVisitor) + } +} diff --git a/src/value/tests.rs b/src/value/tests.rs index 57ba0af9..b62d51cd 100644 --- a/src/value/tests.rs +++ b/src/value/tests.rs @@ -13,11 +13,11 @@ clippy::pattern_type_mismatch )] -use alloc::collections::BTreeMap; +use alloc::collections::{BTreeMap, BTreeSet}; use alloc::format; use alloc::vec::Vec; -use super::Object; +use super::{Object, Set}; use crate::value::Value; fn val(i: u64) -> Value { @@ -562,3 +562,194 @@ fn object_insert_returns_previous_value() { assert_eq!(obj.insert(val(0), val(2)), Some(val(1))); assert_eq!(obj.get(&val(0)), Some(&val(2))); } + +// ========================================================================= +// Set tests +// ========================================================================= + +const SET_SIZES: &[u64] = &[0, 1, 2, 4, 8, 64, 256, 1024]; + +#[test] +fn set_iter_sorted_matches_btreeset_oracle() { + for &n in SET_SIZES { + let values: Vec = (0..n).map(val).collect(); + let oracle: BTreeSet = values.iter().cloned().collect(); + let s: Set = values.into_iter().collect(); + let actual: Vec<&Value> = s.iter_sorted().collect(); + let expected: Vec<&Value> = oracle.iter().collect(); + assert_eq!(actual, expected, "size {n}"); + } +} + +#[test] +fn set_iter_multiset_equality_with_oracle() { + for &n in SET_SIZES { + let values: Vec = (0..n).map(val).collect(); + let oracle: BTreeSet = values.iter().cloned().collect(); + let s: Set = values.into_iter().collect(); + let mut a: Vec = s.iter().cloned().collect(); + let mut b: Vec = oracle.iter().cloned().collect(); + a.sort(); + b.sort(); + assert_eq!(a, b); + } +} + +#[test] +fn set_algebra_matches_btreeset() { + let a_vals: Vec = (0..32_u64).map(val).collect(); + let b_vals: Vec = (16..48_u64).map(val).collect(); + let a_btree: BTreeSet = a_vals.iter().cloned().collect(); + let b_btree: BTreeSet = b_vals.iter().cloned().collect(); + let a: Set = a_vals.into_iter().collect(); + let b: Set = b_vals.into_iter().collect(); + + fn sorted<'a, I: Iterator>(it: I) -> Vec<&'a Value> { + let mut v: Vec<&Value> = it.collect(); + v.sort(); + v + } + + let inter_set = a.intersection(&b); + assert_eq!( + sorted(inter_set.iter_sorted()), + sorted(a_btree.intersection(&b_btree)) + ); + + let union_set = a.union(&b); + assert_eq!( + sorted(union_set.iter_sorted()), + sorted(a_btree.union(&b_btree)) + ); + + let diff_set = a.difference(&b); + assert_eq!( + sorted(diff_set.iter_sorted()), + sorted(a_btree.difference(&b_btree)) + ); + + // Subset: trivial + non-trivial cases. + let proper_subset: Set = (0..16_u64).map(val).collect(); + let non_subset: Set = (30..50_u64).map(val).collect(); + assert!(a.is_subset(&a)); + assert!(proper_subset.is_subset(&a)); + assert!(!non_subset.is_subset(&a)); +} + +#[test] +fn set_first_last() { + let s: Set = (0..16_u64).map(val).collect(); + assert_eq!(s.first(), Some(&val(0))); + assert_eq!(s.last(), Some(&val(15))); + assert!(Set::new().first().is_none()); +} + +#[test] +fn set_serde_roundtrip() { + for &n in &[0_u64, 1, 8, 64] { + let s: Set = (0..n).map(val).collect(); + let json = serde_json::to_string(&s).expect("ser"); + let back: Set = serde_json::from_str(&json).expect("de"); + assert_eq!(s, back, "size {n}"); + } +} + +#[test] +fn set_append_drains_other() { + let mut a: Set = (0..4_u64).map(val).collect(); + let mut b: Set = (4..8_u64).map(val).collect(); + a.append(&mut b); + assert_eq!(a.len(), 8); + assert!(b.is_empty()); +} + +#[test] +fn set_value_cow_make_mut_isolates_clones() { + let a = Value::new_set(); + let b = a.clone(); + let mut b_owned = b; + b_owned.as_set_mut().expect("set").insert(Value::from("x")); + assert_eq!(a.as_set().expect("set").len(), 0); + assert_eq!(b_owned.as_set().expect("set").len(), 1); +} + +#[test] +fn set_from_iter_dedups_duplicates() { + let s: Set = [val(1), val(1), val(2), val(2), val(2)] + .into_iter() + .collect(); + assert_eq!(s.len(), 2); + assert!(s.contains(&val(1))); + assert!(s.contains(&val(2))); +} + +#[test] +fn set_accessor_coverage() { + let mut s: Set = (0..4_u64).map(val).collect(); + + assert!(s.contains(&val(2))); + assert!(!s.contains(&val(100))); + + assert_eq!(s.get(&val(2)), Some(&val(2))); + assert!(s.get(&val(100)).is_none()); + + assert!(s.remove(&val(2))); + assert!(!s.remove(&val(2))); + assert_eq!(s.len(), 3); + + s.retain(|v| v != &val(0)); + assert!(!s.contains(&val(0))); + assert_eq!(s.len(), 2); + + s.clear(); + assert!(s.is_empty()); + assert!(!s.contains(&val(1))); +} + +#[test] +fn set_into_iterator_ref() { + let s: Set = (0..4_u64).map(val).collect(); + let mut count = 0; + for _v in &s { + count += 1; + } + assert_eq!(count, 4); +} + +#[test] +fn set_cursor_yields_every_element_once() { + for &n in SET_SIZES { + let vals: Vec = (0..n).map(val).collect(); + let s: Set = vals.clone().into_iter().collect(); + let mut cursor = s.cursor(); + let mut collected: Vec = Vec::new(); + while let Some(v) = s.next(&mut cursor) { + collected.push(v.clone()); + } + let mut a = collected; + a.sort(); + let mut b = vals; + b.sort(); + assert_eq!(a, b, "size {n}"); + } +} + +#[test] +fn set_cursor_empty_returns_none_immediately() { + let s = Set::new(); + let mut c = s.cursor(); + assert!(s.next(&mut c).is_none()); +} + +#[test] +fn set_ord_invariant_to_insertion_order() { + let mut a = Set::new(); + let mut b = Set::new(); + for i in 0..16_u64 { + a.insert(val(i)); + } + for i in (0..16_u64).rev() { + b.insert(val(i)); + } + assert_eq!(a.cmp(&b), core::cmp::Ordering::Equal); +}