diff --git a/Cargo.lock b/Cargo.lock index 490568cfeb..a3624a2a2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1424,11 +1424,13 @@ dependencies = [ "generic-tests", "hex", "itertools 0.10.5", + "num-rational", "rand 0.8.5", "serde", "serde-big-array", "sha3", "tap", + "test-case", "tracing", "typenum", "zeroize", @@ -3604,6 +3606,39 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" +[[package]] +name = "test-case" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" +dependencies = [ + "test-case-macros", +] + +[[package]] +name = "test-case-core" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" +dependencies = [ + "cfg-if 1.0.4", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "test-case-macros" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "test-case-core", +] + [[package]] name = "test-strategy" version = "0.3.1" diff --git a/fastcrypto-tbls/Cargo.toml b/fastcrypto-tbls/Cargo.toml index 6ce99b7a31..036dd8065d 100644 --- a/fastcrypto-tbls/Cargo.toml +++ b/fastcrypto-tbls/Cargo.toml @@ -10,6 +10,7 @@ repository = "https://github.com/MystenLabs/fastcrypto" [dependencies] fastcrypto = { path = "../fastcrypto", features = ["aes"] } +num-rational = { version = "0.4.2", features = ["std"] } rand.workspace = true serde.workspace = true @@ -28,6 +29,7 @@ serde-big-array = "0.5.1" [dev-dependencies] criterion = "0.5.1" generic-tests = "0.1.2" +test-case = "3.3.1" [[bench]] name = "polynomial" diff --git a/fastcrypto-tbls/src/lib.rs b/fastcrypto-tbls/src/lib.rs index 6068fc25be..7ec137820b 100644 --- a/fastcrypto-tbls/src/lib.rs +++ b/fastcrypto-tbls/src/lib.rs @@ -20,6 +20,7 @@ pub mod polynomial; pub mod tbls; pub mod threshold_schnorr; pub mod types; +pub mod weight_reduction; // TODO: needs to use ecies_v1 // #[cfg(any(test, feature = "experimental"))] @@ -58,4 +59,8 @@ pub mod nizk; #[cfg(test)] #[path = "tests/nizk_tests.rs"] pub mod nizk_tests; + pub mod random_oracle; +#[cfg(test)] +#[path = "tests/super_swiper_test.rs"] +pub mod super_swiper_test; diff --git a/fastcrypto-tbls/src/nodes.rs b/fastcrypto-tbls/src/nodes.rs index 4e9fc6d0fd..24fd138569 100644 --- a/fastcrypto-tbls/src/nodes.rs +++ b/fastcrypto-tbls/src/nodes.rs @@ -3,14 +3,24 @@ use crate::ecies_v1; use crate::types::ShareIndex; +use crate::weight_reduction::solve; +use crate::weight_reduction::weight_reduction_checks::compute_precision_loss; use fastcrypto::error::{FastCryptoError, FastCryptoResult}; use fastcrypto::groups::GroupElement; use fastcrypto::hash::{Blake2b256, Digest, HashFunction}; +use itertools::Itertools; +use num_rational::Ratio; use serde::{Deserialize, Serialize}; use tracing::debug; pub type PartyId = u16; +/// Best reduction candidate: per-party weights, reduced total weight W', precision loss δ, divisor d. +type ReductionBest = (Vec, u64, Ratio, Ratio); + +/// Best super_swiper candidate: reduced total weight W', per-party weights, precision loss δ, divisor d. +type SuperSwiperBest = (u64, Vec, Ratio, Ratio); + /// Public parameters of a party. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct Node { @@ -223,6 +233,158 @@ impl Nodes { )) } + /// Create a new set of nodes using the super_swiper algorithm under the **bilateral** + /// weight-reduction analysis (Appendix `weights-bilateral.tex`). + /// + /// Stage 1 (search). Outer loop: α from 0.10 to 0.90 in steps of 1/100. Inner loop: + /// β from α + 0.01 to α + 0.20 in steps of 1/100 (skip β ≥ 1). Pass each (α, β) to + /// [`solve`](crate::weight_reduction::solve). Among candidates with W' ≥ + /// `total_weight_lower_bound` that satisfy the bilateral feasibility criterion + /// + /// ```text + /// 3·δ + d ≤ allowed_delta, + /// ``` + /// + /// keep the reduction with smallest W' (tie-break: smaller δ when W' is equal). + /// Here `δ = Σ_i max(w_i − w'_i·d, 0)` and `d = W/W'` (exact ratios). + /// + /// Stage 2 (closed form). Given the original-space pair `(t, f)` (with `t > f` and + /// `t + 2f ≤ W`), + /// + /// ```text + /// t' = ⌈ (t + δ) / d ⌉, + /// f' = ⌊ (f + δ) / d ⌋. + /// ``` + /// + /// Under these `(t', f')`, Theorem (Safety, Liveness, Byzantine removal) of the bilateral + /// analysis applies: a coalition with original weight ≥ `t + f + allowed_delta` clears the + /// reduced-space target `t' + f'`, while any Byzantine subset bounded by `f` in original + /// space contributes at most `f'` in the reduced space. + /// + /// # Parameters + /// - `nodes_vec`: Input nodes with original weights. + /// - `t`: Original-space safety threshold. + /// - `f`: Original-space Byzantine bound (`t > f`, `t + 2f ≤ W`). + /// - `allowed_delta`: Stage-1 budget bounding `3·δ + d`. + /// - `total_weight_lower_bound`: Minimum allowed reduced total weight `W'`. + /// + /// # Returns + /// `(reduced Nodes, t', f')`, or `Err` if no Stage-1 candidate satisfies the criterion. + pub fn new_super_swiper_reduced( + nodes_vec: Vec>, + t: u16, + f: u16, + allowed_delta: u16, + total_weight_lower_bound: u16, + ) -> FastCryptoResult<(Self, u16, u16)> { + let n = Self::new(nodes_vec)?; + let original_total_weight = n.total_weight() as u64; + + if total_weight_lower_bound > n.total_weight + || total_weight_lower_bound == 0 + || original_total_weight == 0 + { + return Err(FastCryptoError::InvalidInput); + } + + let weights_sorted = n + .nodes + .iter() + .map(|node| node.weight as u64) + .sorted() + .rev() + .collect_vec(); + + let original_weights: Vec = n.nodes.iter().map(|node| node.weight as u64).collect(); + + let indexed_weights: Vec<(usize, u16)> = n + .nodes + .iter() + .enumerate() + .map(|(i, node)| (i, node.weight)) + .sorted_by_key(|(_, w)| *w) + .rev() + .collect_vec(); + + // Stage 1: search (α, β) and keep the smallest W' (tie-break: smallest δ) satisfying 3δ + d ≤ allowed_delta. + let allowed_delta_ratio = Ratio::from_integer(allowed_delta as u64); + let three = Ratio::from_integer(3u64); + let one = Ratio::from_integer(1u64); + let (new_total_weight, new_weights, delta, d) = { + let mut best: Option = None; + for a_numer in 10u64..=90u64 { + let alpha = Ratio::new(a_numer, 100); + for b_extra_numer in 1u64..=20u64 { + let beta = alpha + Ratio::new(b_extra_numer, 100); + if beta >= one { + continue; + } + let reduced_weights_sorted = solve(alpha, beta, &weights_sorted); + let new_total_weight: u64 = reduced_weights_sorted.iter().sum(); + if new_total_weight < total_weight_lower_bound as u64 { + continue; + } + let mut new_weights = vec![0u16; n.nodes.len()]; + for (idx_in_sorted, (original_idx, _)) in indexed_weights.iter().enumerate() { + if idx_in_sorted < reduced_weights_sorted.len() { + new_weights[*original_idx] = + reduced_weights_sorted[idx_in_sorted] as u16; + } + } + let reduced_weights: Vec = + new_weights.iter().copied().map(u64::from).collect(); + let (delta, d) = compute_precision_loss(&original_weights, &reduced_weights); + if three * delta + d <= allowed_delta_ratio { + let take = match &best { + None => true, + Some((best_w, _, best_delta, _)) => { + new_total_weight < *best_w + || (new_total_weight == *best_w && delta < *best_delta) + } + }; + if take { + best = Some((new_total_weight, new_weights, delta, d)); + } + } + } + } + best.ok_or(FastCryptoError::InvalidInput)? + }; + + // Stage 2: t' = ⌈(t + δ)/d⌉, f' = ⌊(f + δ)/d⌋. + let t_prime_int = ((Ratio::from_integer(t as u64) + delta) / d) + .ceil() + .to_integer(); + let f_prime_int = ((Ratio::from_integer(f as u64) + delta) / d) + .floor() + .to_integer(); + + let nodes = n + .nodes + .into_iter() + .zip(new_weights) + .map(|(Node { id, pk, weight: _ }, new_weight)| Node { + id, + pk, + weight: new_weight, + }) + .collect_vec(); + + let accumulated_weights = Self::get_accumulated_weights(&nodes); + let nodes_with_nonzero_weight = Self::filter_nonzero_weights(&nodes); + + Ok(( + Self { + nodes, + total_weight: new_total_weight as u16, + accumulated_weights, + nodes_with_nonzero_weight, + }, + t_prime_int as u16, + f_prime_int as u16, + )) + } + /// Create a new set of nodes. Nodes must have consecutive ids starting from 0. /// Reduces weights up to an allowed delta in the original total weight. /// Finds the largest d such that: @@ -255,8 +417,9 @@ impl Nodes { } // Compute the precision loss. // U16 is safe here since total_weight is u16. - let delta = - n.nodes.iter().map(|n| n.weight % d).sum::() + neg_mod(t, d) + neg_mod(f, d); + let delta = n.nodes.iter().map(|n| n.weight % d).sum::() + + Self::neg_mod(t, d) + + Self::neg_mod(f, d); if delta <= allowed_delta { max_d = d; } @@ -292,9 +455,122 @@ impl Nodes { new_f, )) } -} -/// Compute (-x) mod d = d * ceil(x/d) - x -fn neg_mod(x: u16, d: u16) -> u16 { - (-(x as i32)).rem_euclid(d as i32) as u16 + /// Weight reduction via floor division with exact precision loss (PROOF.md §Unilateral). + /// + /// Loops `div` from 100 down to 2, computing `w'_i = floor(w_i / div)` and exact + /// (δ, d) via [`compute_precision_loss`]. Keeps the solution with smallest W' whose + /// δ ≤ `allowed_delta`. Stops as soon as δ > `allowed_delta` (larger div ⇒ larger δ). + /// + /// Thresholds follow the unilateral formulas: + /// - `t' = t_min / d` + /// - `f' = (L - t_min - δ) / d` + /// + /// # Parameters + /// - `nodes_vec`: Input nodes with weights + /// - `t_min`: Safety threshold lower bound in original weight space + /// - `liveness_upper_bound`: `L` — upper bound in original space for `t' + f'` + /// - `allowed_delta`: Maximum acceptable precision loss δ + /// - `total_weight_lower_bound`: Minimum allowed total weight after reduction + /// + /// # Returns + /// `(reduced Nodes, t', f')` or error if no feasible reduction exists. + pub fn new_reduced_v2( + nodes_vec: Vec>, + t_min: u16, + liveness_upper_bound: u16, + allowed_delta: u16, + total_weight_lower_bound: u16, + ) -> FastCryptoResult<(Self, u16, u16)> { + let n = Self::new(nodes_vec)?; + if total_weight_lower_bound > n.total_weight + || total_weight_lower_bound == 0 + || n.total_weight == 0 + { + return Err(FastCryptoError::InvalidInput); + } + + let original_weights: Vec = n.nodes.iter().map(|node| node.weight as u64).collect(); + let allowed_delta_ratio = Ratio::from_integer(allowed_delta as u64); + + // Try a candidate divisor: w'_i = floor(w_i * scale / denom) where div = denom/scale. + // For coarse steps scale=1, denom=div. For fine steps scale=100, denom=div*100+k. + let try_div = |scale: u64, denom: u64| -> Option { + let new_weights: Vec = original_weights + .iter() + .map(|&w| (w * scale / denom) as u16) + .collect(); + let new_total_weight: u64 = new_weights.iter().map(|&w| u64::from(w)).sum(); + if new_total_weight < u64::from(total_weight_lower_bound) { + return None; + } + let reduced: Vec = new_weights.iter().copied().map(u64::from).collect(); + let (delta, d) = compute_precision_loss(&original_weights, &reduced); + Some((new_weights, new_total_weight, delta, d)) + }; + + let mut best: Option = None; + for div in 2..=100u64 { + if let Some((weights, w_prime, delta, d)) = try_div(1, div) { + if delta <= allowed_delta_ratio { + best = Some((weights, w_prime, delta, d)); + continue; + } + } + // delta exceeded budget or W' too small — fine-scan the gap [div-1, div) in 0.01 steps. + // div = (div-1)*100 + k) / 100 for k = 1..99. + let base = (div - 1) * 100; + for k in 1..100u64 { + if let Some((weights, w_prime, delta, d)) = try_div(100, base + k) { + if delta <= allowed_delta_ratio { + best = Some((weights, w_prime, delta, d)); + } else { + break; + } + } + } + break; + } + + let (new_weights, new_total_weight, delta, d) = + best.ok_or(FastCryptoError::InvalidInput)?; + + // t' = t_min / d (unilateral, PROOF.md) + let t_min_ratio = Ratio::from_integer(u64::from(t_min)); + let new_t = (t_min_ratio / d).to_integer() as u16; + + // f' = (L - t_min - δ) / d (unilateral, PROOF.md) + let l_ratio = Ratio::from_integer(u64::from(liveness_upper_bound)); + let new_f = ((l_ratio - t_min_ratio - delta) / d).to_integer() as u16; + + let nodes = n + .nodes + .into_iter() + .zip(new_weights) + .map(|(Node { id, pk, weight: _ }, new_weight)| Node { + id, + pk, + weight: new_weight, + }) + .collect::>(); + + let accumulated_weights = Self::get_accumulated_weights(&nodes); + let nodes_with_nonzero_weight = Self::filter_nonzero_weights(&nodes); + + Ok(( + Self { + nodes, + total_weight: new_total_weight as u16, + accumulated_weights, + nodes_with_nonzero_weight, + }, + new_t, + new_f, + )) + } + + /// Compute (-x) mod d = d * ceil(x/d) - x + fn neg_mod(x: u16, d: u16) -> u16 { + (-(x as i32)).rem_euclid(d as i32) as u16 + } } diff --git a/fastcrypto-tbls/src/tests/super_swiper_test.rs b/fastcrypto-tbls/src/tests/super_swiper_test.rs new file mode 100644 index 0000000000..1321b749a5 --- /dev/null +++ b/fastcrypto-tbls/src/tests/super_swiper_test.rs @@ -0,0 +1,1263 @@ +// Copyright (c) 2022, Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Epoch comparison tests for `Nodes::new_super_swiper_reduced` (bilateral analysis, +// `weights-bilateral.tex`), `Nodes::new_reduced`, `Nodes::new_reduced_with_f`, and +// `Nodes::new_reduced_v2`. +// Baseline: t=34%·W, L=75%·W, allowed_delta=8%·W. Alt: t=52%·W, L=80%·W, allowed_delta=8%·W. +// Run: cargo test -p fastcrypto-tbls --lib super_swiper_test::tests -- --nocapture + +mod tests { + use crate::ecies_v1; + use crate::nodes::{Node, Nodes}; + use crate::weight_reduction::weight_reduction_checks::compute_precision_loss; + use fastcrypto::error::FastCryptoResult; + + use fastcrypto::groups::ristretto255::RistrettoPoint; + use fastcrypto::groups::{FiatShamirChallenge, GroupElement}; + use num_rational::Ratio; + use rand::thread_rng; + use serde::de::DeserializeOwned; + use serde::Serialize; + use zeroize::Zeroize; + + fn create_test_nodes(weights: Vec) -> Vec> + where + G: GroupElement + Serialize + DeserializeOwned, + G::ScalarType: FiatShamirChallenge + Zeroize, + { + let sk = ecies_v1::PrivateKey::::new(&mut thread_rng()); + let pk = ecies_v1::PublicKey::::from_private_key(&sk); + weights + .into_iter() + .enumerate() + .map(|(i, weight)| Node { + id: i as u16, + pk: pk.clone(), + weight, + }) + .collect() + } + + // Helper function to load Sui validator voting power for a specific epoch + fn load_sui_validator_voting_power_for_epoch(epoch: u64) -> Vec { + let weights_data = match epoch { + 100 => include_str!("../weight_reduction/data/sui_real_all_voting_power_epoch_100.dat"), + 200 => include_str!("../weight_reduction/data/sui_real_all_voting_power_epoch_200.dat"), + 400 => include_str!("../weight_reduction/data/sui_real_all_voting_power_epoch_400.dat"), + 800 => include_str!("../weight_reduction/data/sui_real_all_voting_power_epoch_800.dat"), + 974 => include_str!("../weight_reduction/data/sui_real_all_voting_power_epoch_974.dat"), + _ => panic!("Unsupported epoch: {}", epoch), + }; + weights_data + .lines() + .map(|line| line.trim()) + .filter(|line| !line.is_empty()) + .map(|line| { + line.parse::() + .unwrap_or_else(|_| panic!("Failed to parse voting power: {}", line)) + }) + .collect() + } + + fn scale_weights_to_u16(weights: &[u64]) -> Vec { + if weights.is_empty() { + return vec![]; + } + // Calculate total weight first to ensure it fits in u16::MAX + let total: u64 = weights.iter().sum(); + let max_weight = *weights.iter().max().unwrap(); + + // Scale factor: we need to ensure both individual weights and total weight fit in u16 + let scale_for_max = if max_weight > u16::MAX as u64 { + (max_weight as f64 / u16::MAX as f64).ceil() as u64 + } else { + 1 + }; + let scale_for_total = if total > u16::MAX as u64 { + (total as f64 / u16::MAX as f64).ceil() as u64 + } else { + 1 + }; + let scale_factor = scale_for_max.max(scale_for_total); + + weights + .iter() + .map(|&w| { + let scaled = (w / scale_factor.max(1)) as u16; + // Ensure at least 1 if original was > 0 + scaled.max(1) + }) + .collect() + } + + fn ratio_to_decimal(r: &Ratio) -> String { + format!("{:.6}", *r.numer() as f64 / *r.denom() as f64) + } + + /// Largest `f` with `t + 2f <= w` and `t > f` (unsigned), or `None` if no such `f` exists. + fn f_upper_bound(w: u64, t: u16) -> Option { + let t_u = u64::from(t); + if t_u == 0 || w < t_u { + None + } else { + let cap_from_sum = (w - t_u) / 2; + let cap_from_strict = t_u - 1; + Some(cap_from_sum.min(cap_from_strict)) + } + } + + // Type alias for epoch comparison results: (epoch, W, t, f, W', t', f', delta, d) + type EpochComparisonResult = ( + u64, + Option, + Option, + Option, + Option, + Option, + Option, + Option, + Option, + ); + + type ReduceRistretto = fn( + Vec>, + u16, + u16, + u16, + ) -> FastCryptoResult<(Nodes, u16)>; + + /// Reducers that take `(t, f)` natively and return `(reduced Nodes, t', f')`. + /// Used for `new_super_swiper_reduced` (bilateral) and `new_reduced_with_f` (shipping). + type ReduceWithFRistretto = fn( + Vec>, + u16, + u16, + u16, + u16, + ) -> FastCryptoResult<(Nodes, u16, u16)>; + + /// Threshold `t`, liveness `L`, and reducer `allowed_delta` as fractions of total weight `W`. + struct EpochChartParams { + /// `t = floor((t_alpha * W).to_integer())` with `t_alpha = t_numer / t_denom`. + t_alpha: Ratio, + /// `L = floor(liveness_pct * W)` as `u16`. + liveness_pct: f64, + /// `allowed_delta = floor(allowed_delta_pct * W)` as `u16`. + allowed_delta_pct: f64, + } + + impl EpochChartParams { + fn baseline_34_75_8() -> Self { + Self { + t_alpha: Ratio::new(34u64, 100u64), + liveness_pct: 0.75, + allowed_delta_pct: 0.08, + } + } + + /// `t = 52% W`, `L = 80% W`, `allowed_delta = 8% W`. + fn t52_l80_delta8() -> Self { + Self { + t_alpha: Ratio::new(52u64, 100u64), + liveness_pct: 0.80, + allowed_delta_pct: 0.08, + } + } + } + + /// How `f_l` and scaled `f'_l` are derived from liveness `L`, threshold `t`, and precision loss + /// for reducers (like `new_reduced`) that don't return `f'` natively. + enum FFromLiveness { + /// `f_l = L - t - δ`, `f'_l = f_l / d` (`new_reduced` convention). + NewReduced, + } + + /// Derive the bilateral-`f` from chart params for reducers that take `f` natively. + /// + /// Choose `f` so that the bilateral liveness statement `w(S) ≥ t + f + δ_allowed` is implied + /// by `w(S) ≥ L`, i.e. `f = L − t − allowed_delta`, capped by feasibility (`t > f`, + /// `t + 2f ≤ W`). + fn f_from_chart(w: u64, t: u16, liveness: u16, allowed_delta: u16) -> u16 { + let f_raw = u64::from(liveness) + .saturating_sub(u64::from(t)) + .saturating_sub(u64::from(allowed_delta)); + let f_capped = f_upper_bound(w, t).map(|cap| f_raw.min(cap)).unwrap_or(0); + u16::try_from(f_capped).unwrap_or(u16::MAX) + } + + fn value_with_pct_of_w_prime(opt: Option, w_prime: Option) -> String { + match (opt, w_prime) { + (Some(n), Some(wp)) if wp > 0 => { + let p = 100.0 * f64::from(n) / f64::from(wp); + format!("{n} ({p:.1}%)") + } + (Some(n), Some(_)) => format!("{n} (N/A)"), + (Some(n), None) => n.to_string(), + (None, _) => "N/A".to_string(), + } + } + + fn run_all_epochs_comparison_chart( + reducer: ReduceRistretto, + chart_title: &str, + fail_label: &str, + f_from_liveness: FFromLiveness, + params: &EpochChartParams, + ) { + let epochs = vec![100, 200, 400, 800, 974]; + let mut results: Vec = Vec::new(); + + println!( + "\n🔬 Testing weight reduction across multiple epochs ({})...\n params: t = {}%·W, L = {}%·W, allowed_delta = {}%·W\n", + chart_title, + 100.0 * (*params.t_alpha.numer() as f64) / (*params.t_alpha.denom() as f64), + params.liveness_pct * 100.0, + params.allowed_delta_pct * 100.0, + ); + + for epoch in epochs { + println!("📊 Processing epoch {}...", epoch); + let sui_weights = load_sui_validator_voting_power_for_epoch(epoch); + let scaled_weights = scale_weights_to_u16(&sui_weights); + let nodes_vec = create_test_nodes::(scaled_weights.clone()); + let original_nodes = Nodes::new(nodes_vec.clone()).unwrap(); + let original_total_weight = original_nodes.total_weight(); + + let t = (params.t_alpha * original_total_weight as u64).to_integer() as u16; + let allowed_delta = (original_total_weight as f64 * params.allowed_delta_pct) as u16; + let total_weight_lower_bound = 1u16; + + let reduced_result = reducer( + nodes_vec.clone(), + t, + allowed_delta, + total_weight_lower_bound, + ); + + let liveness_weight = (original_total_weight as f64 * params.liveness_pct) as u16; + let (w_prime, t_prime, f_prime, delta_str, d_str, f) = match reduced_result { + Ok((reduced_nodes, new_t)) => { + let w_p = reduced_nodes.total_weight(); + let original_weights: Vec = + scaled_weights.iter().map(|&w| w as u64).collect(); + let reduced_weights: Vec = + reduced_nodes.iter().map(|n| n.weight as u64).collect(); + let (precision_delta, d) = + compute_precision_loss(&original_weights, &reduced_weights); + let delta_int = precision_delta.to_integer(); + let d_int = d.to_integer().max(1); + + let w = u64::from(original_total_weight); + let f_l = match f_from_liveness { + FFromLiveness::NewReduced => { + u64::from(liveness_weight).saturating_sub(u64::from(t) + delta_int) + } + }; + let f_val = f_upper_bound(w, t).map(|cap| f_l.min(cap)); + + let f_prime_l = match f_from_liveness { + FFromLiveness::NewReduced => f_l / d_int, + }; + let f_prime_val = f_upper_bound(u64::from(w_p), new_t) + .and_then(|cap| u16::try_from(f_prime_l.min(cap)).ok()); + + let delta_val = Some(ratio_to_decimal(&precision_delta)); + let d_val = Some(ratio_to_decimal(&d)); + (Some(w_p), Some(new_t), f_prime_val, delta_val, d_val, f_val) + } + Err(_) => ( + None::, + None::, + None::, + None::, + None::, + None::, + ), + }; + + let row: EpochComparisonResult = ( + epoch, + Some(original_total_weight), + Some(t), + f, + w_prime, + t_prime, + f_prime, + delta_str.clone(), + d_str.clone(), + ); + results.push(row); + + println!( + " ✅ Epoch {}: W={}, t={}, f={:?}, W'={:?}, t'={:?}, f'={:?}, δ={:?}, d={:?}", + epoch, original_total_weight, t, f, w_prime, t_prime, f_prime, delta_str, d_str + ); + } + + let col_tpf = 20; + let separator = "=".repeat(120); + println!("\n{}", separator); + println!( + "📈 WEIGHT REDUCTION COMPARISON CHART — {} (t={}%·W, L={}%·W, δ_allow={}%·W)", + chart_title, + 100.0 * (*params.t_alpha.numer() as f64) / (*params.t_alpha.denom() as f64), + params.liveness_pct * 100.0, + params.allowed_delta_pct * 100.0, + ); + println!("{}", separator); + println!( + "{:<8} | {:<6} | {:<6} | {:<8} | {:<8} | {: = Vec::new(); + + println!( + "\n🔬 Testing weight reduction across multiple epochs ({})...\n params: t = {}%·W, L = {}%·W, allowed_delta = {}%·W\n", + chart_title, + 100.0 * (*params.t_alpha.numer() as f64) / (*params.t_alpha.denom() as f64), + params.liveness_pct * 100.0, + params.allowed_delta_pct * 100.0, + ); + + for epoch in epochs { + println!("📊 Processing epoch {}...", epoch); + let sui_weights = load_sui_validator_voting_power_for_epoch(epoch); + let scaled_weights = scale_weights_to_u16(&sui_weights); + let nodes_vec = create_test_nodes::(scaled_weights.clone()); + let original_nodes = Nodes::new(nodes_vec.clone()).unwrap(); + let original_total_weight = original_nodes.total_weight(); + + let t = (params.t_alpha * original_total_weight as u64).to_integer() as u16; + let liveness_weight = (original_total_weight as f64 * params.liveness_pct) as u16; + let allowed_delta = (original_total_weight as f64 * params.allowed_delta_pct) as u16; + let f = f_from_chart( + u64::from(original_total_weight), + t, + liveness_weight, + allowed_delta, + ); + let total_weight_lower_bound = 1u16; + + let reduced_result = reducer( + nodes_vec.clone(), + t, + f, + allowed_delta, + total_weight_lower_bound, + ); + + let (w_prime, t_prime, f_prime, delta_str, d_str, f_orig) = match reduced_result { + Ok((reduced_nodes, new_t, new_f)) => { + let w_p = reduced_nodes.total_weight(); + let original_weights: Vec = + scaled_weights.iter().map(|&w| w as u64).collect(); + let reduced_weights: Vec = + reduced_nodes.iter().map(|n| n.weight as u64).collect(); + let (precision_delta, d) = + compute_precision_loss(&original_weights, &reduced_weights); + let delta_val = Some(ratio_to_decimal(&precision_delta)); + let d_val = Some(ratio_to_decimal(&d)); + ( + Some(w_p), + Some(new_t), + Some(new_f), + delta_val, + d_val, + Some(u64::from(f)), + ) + } + Err(_) => ( + None::, + None::, + None::, + None::, + None::, + None::, + ), + }; + + let row: EpochComparisonResult = ( + epoch, + Some(original_total_weight), + Some(t), + f_orig, + w_prime, + t_prime, + f_prime, + delta_str.clone(), + d_str.clone(), + ); + results.push(row); + + println!( + " ✅ Epoch {}: W={}, t={}, f={}, W'={:?}, t'={:?}, f'={:?}, δ={:?}, d={:?}", + epoch, original_total_weight, t, f, w_prime, t_prime, f_prime, delta_str, d_str + ); + } + + let col_tpf = 20; + let separator = "=".repeat(120); + println!("\n{}", separator); + println!( + "📈 WEIGHT REDUCTION COMPARISON CHART — {} (t={}%·W, L={}%·W, δ_allow={}%·W)", + chart_title, + 100.0 * (*params.t_alpha.numer() as f64) / (*params.t_alpha.denom() as f64), + params.liveness_pct * 100.0, + params.allowed_delta_pct * 100.0, + ); + println!("{}", separator); + println!( + "{:<8} | {:<6} | {:<6} | {:<8} | {:<8} | {:, + t_prime: Option, + f_prime: Option, + delta: Option, + d: Option, + } + + impl AlgRow { + fn empty() -> Self { + Self { + w_prime: None, + t_prime: None, + f_prime: None, + delta: None, + d: None, + } + } + } + + struct EpochRow { + epoch: u64, + w: u16, + t: u16, + f: u16, + allowed_delta: u16, + ss: AlgRow, + nrwf: AlgRow, + } + + println!( + "\n🔬 super_swiper_reduced (bilateral) vs new_reduced_with_f across multiple epochs…\n params: t = {}%·W, L = {}%·W, allowed_delta = {}%·W\n", + 100.0 * (*params.t_alpha.numer() as f64) / (*params.t_alpha.denom() as f64), + params.liveness_pct * 100.0, + params.allowed_delta_pct * 100.0, + ); + + let run_one = |reduced: FastCryptoResult<(Nodes, u16, u16)>, + scaled_weights: &[u16]| + -> AlgRow { + match reduced { + Ok((reduced_nodes, t_p, f_p)) => { + let w_p = reduced_nodes.total_weight(); + let original_weights: Vec = + scaled_weights.iter().map(|&w| w as u64).collect(); + let reduced_weights: Vec = + reduced_nodes.iter().map(|n| n.weight as u64).collect(); + let (delta, d) = compute_precision_loss(&original_weights, &reduced_weights); + AlgRow { + w_prime: Some(w_p), + t_prime: Some(t_p), + f_prime: Some(f_p), + delta: Some(ratio_to_decimal(&delta)), + d: Some(ratio_to_decimal(&d)), + } + } + Err(_) => AlgRow::empty(), + } + }; + + let mut rows: Vec = Vec::new(); + for epoch in &epochs { + println!("📊 Processing epoch {}...", epoch); + let sui_weights = load_sui_validator_voting_power_for_epoch(*epoch); + let scaled_weights = scale_weights_to_u16(&sui_weights); + let nodes_vec = create_test_nodes::(scaled_weights.clone()); + let original_nodes = Nodes::new(nodes_vec.clone()).unwrap(); + let w = original_nodes.total_weight(); + + let t = (params.t_alpha * w as u64).to_integer() as u16; + let liveness_weight = (w as f64 * params.liveness_pct) as u16; + let allowed_delta = (w as f64 * params.allowed_delta_pct) as u16; + let f = f_from_chart(u64::from(w), t, liveness_weight, allowed_delta); + let total_weight_lower_bound = 1u16; + + let ss = run_one( + Nodes::new_super_swiper_reduced( + nodes_vec.clone(), + t, + f, + allowed_delta, + total_weight_lower_bound, + ), + &scaled_weights, + ); + let nrwf = run_one( + Nodes::new_reduced_with_f(nodes_vec, t, f, allowed_delta, total_weight_lower_bound), + &scaled_weights, + ); + + println!( + " ✅ Epoch {}: W={}, t={}, f={}, allowed_delta={}", + epoch, w, t, f, allowed_delta + ); + println!( + " super_swiper: W'={:?}, t'={:?}, f'={:?}, δ={:?}, d={:?}", + ss.w_prime, ss.t_prime, ss.f_prime, ss.delta, ss.d + ); + println!( + " new_reduced_with_f: W'={:?}, t'={:?}, f'={:?}, δ={:?}, d={:?}", + nrwf.w_prime, nrwf.t_prime, nrwf.f_prime, nrwf.delta, nrwf.d + ); + + rows.push(EpochRow { + epoch: *epoch, + w, + t, + f, + allowed_delta, + ss, + nrwf, + }); + } + + let col_w = 8; + let col_tpf = 12; + let col_dlt = 10; + let line_width = 8 + + 3 + + col_w + + 3 + + col_w + + 3 + + col_w + + 3 + + col_w + + 3 + + col_w + + 3 + + col_tpf + + 3 + + col_tpf + + 3 + + col_dlt + + 3 + + col_dlt + + 3 + + col_w + + 3 + + col_tpf + + 3 + + col_tpf + + 3 + + col_dlt + + 3 + + col_dlt; + let sep = "=".repeat(line_width); + println!("\n{}", sep); + println!( + "📈 super_swiper_reduced (bilateral) vs new_reduced_with_f — t={}%·W, L={}%·W, δ_allow={}%·W", + 100.0 * (*params.t_alpha.numer() as f64) / (*params.t_alpha.denom() as f64), + params.liveness_pct * 100.0, + params.allowed_delta_pct * 100.0, + ); + println!("{}", sep); + + // Header row 1: algorithm group labels. + println!( + "{:<8} | {:| s.clone().unwrap_or_else(|| "N/A".to_string()); + let num_or = |x: Option| { + x.map(|v| v.to_string()) + .unwrap_or_else(|| "N/A".to_string()) + }; + for r in &rows { + println!( + "{:<8} | {: = Vec::new(); + + println!( + "\n🔬 Testing new_reduced_v2 across multiple epochs ({})...\n params: t_min = {}%·W, L = {}%·W, allowed_delta = {}%·W\n", + chart_title, + 100.0 * (*params.t_alpha.numer() as f64) / (*params.t_alpha.denom() as f64), + params.liveness_pct * 100.0, + params.allowed_delta_pct * 100.0, + ); + + for epoch in epochs { + println!("📊 Processing epoch {}...", epoch); + let sui_weights = load_sui_validator_voting_power_for_epoch(epoch); + let scaled_weights = scale_weights_to_u16(&sui_weights); + let nodes_vec = create_test_nodes::(scaled_weights.clone()); + let original_nodes = Nodes::new(nodes_vec.clone()).unwrap(); + let original_total_weight = original_nodes.total_weight(); + + let t_min = (params.t_alpha * original_total_weight as u64).to_integer() as u16; + let liveness_upper_bound = (original_total_weight as f64 * params.liveness_pct) as u16; + let allowed_delta = (original_total_weight as f64 * params.allowed_delta_pct) as u16; + let total_weight_lower_bound = 1u16; + + let reduced_result = Nodes::new_reduced_v2( + nodes_vec.clone(), + t_min, + liveness_upper_bound, + allowed_delta, + total_weight_lower_bound, + ); + + let (w_prime, t_prime, f_prime, delta_str, d_str, f) = match reduced_result { + Ok((reduced_nodes, new_t, new_f)) => { + let w_p = reduced_nodes.total_weight(); + let original_weights: Vec = + scaled_weights.iter().map(|&w| w as u64).collect(); + let reduced_weights: Vec = + reduced_nodes.iter().map(|n| n.weight as u64).collect(); + let (precision_delta, d) = + compute_precision_loss(&original_weights, &reduced_weights); + // f = f' * d (unilateral, PROOF.md) + let d_int = d.to_integer().max(1); + let f_val = u64::from(new_f) * d_int; + + let delta_val = Some(ratio_to_decimal(&precision_delta)); + let d_val = Some(ratio_to_decimal(&d)); + ( + Some(w_p), + Some(new_t), + Some(new_f), + delta_val, + d_val, + Some(f_val), + ) + } + Err(_) => ( + None::, + None::, + None::, + None::, + None::, + None::, + ), + }; + + let row: EpochComparisonResult = ( + epoch, + Some(original_total_weight), + Some(t_min), + f, + w_prime, + t_prime, + f_prime, + delta_str.clone(), + d_str.clone(), + ); + results.push(row); + + println!( + " ✅ Epoch {}: W={}, t_min={}, f={:?}, W'={:?}, t'={:?}, f'={:?}, δ={:?}, d={:?}", + epoch, original_total_weight, t_min, f, w_prime, t_prime, f_prime, delta_str, d_str + ); + } + + let col_tpf = 20; + let separator = "=".repeat(120); + println!("\n{}", separator); + println!( + "📈 WEIGHT REDUCTION COMPARISON CHART — {} (t_min={}%·W, L={}%·W, δ_allow={}%·W)", + chart_title, + 100.0 * (*params.t_alpha.numer() as f64) / (*params.t_alpha.denom() as f64), + params.liveness_pct * 100.0, + params.allowed_delta_pct * 100.0, + ); + println!("{}", separator); + println!( + "{:<8} | {:<6} | {:<6} | {:<8} | {:<8} | {:, + w_prime_super_swiper: Option, + w_prime_new_reduced_v2: Option, + } + + println!( + "\n🔬 W vs W' by algorithm (all epochs)…\n params: t = {}%·W, L = {}%·W, allowed_delta = {}%·W\n", + 100.0 * (*params.t_alpha.numer() as f64) / (*params.t_alpha.denom() as f64), + params.liveness_pct * 100.0, + params.allowed_delta_pct * 100.0, + ); + + let mut rows: Vec = Vec::new(); + for epoch in &epochs { + println!("📊 Processing epoch {}...", epoch); + let sui_weights = load_sui_validator_voting_power_for_epoch(*epoch); + let scaled_weights = scale_weights_to_u16(&sui_weights); + let nodes_vec = create_test_nodes::(scaled_weights); + let original_nodes = Nodes::new(nodes_vec.clone()).unwrap(); + let w = original_nodes.total_weight(); + + let t = (params.t_alpha * w as u64).to_integer() as u16; + let liveness_upper_bound = (w as f64 * params.liveness_pct) as u16; + let allowed_delta = (w as f64 * params.allowed_delta_pct) as u16; + let total_weight_lower_bound = 1u16; + + let w_prime_new_reduced = Nodes::new_reduced( + nodes_vec.clone(), + t, + allowed_delta, + total_weight_lower_bound, + ) + .ok() + .map(|(n, _)| n.total_weight()); + + // Bilateral super_swiper takes (t, f) natively; derive `f` from chart liveness. + let f = f_from_chart(u64::from(w), t, liveness_upper_bound, allowed_delta); + let w_prime_super_swiper = Nodes::new_super_swiper_reduced( + nodes_vec.clone(), + t, + f, + allowed_delta, + total_weight_lower_bound, + ) + .ok() + .map(|(n, _, _)| n.total_weight()); + + let w_prime_new_reduced_v2 = Nodes::new_reduced_v2( + nodes_vec, + t, + liveness_upper_bound, + allowed_delta, + total_weight_lower_bound, + ) + .ok() + .map(|(n, _, _)| n.total_weight()); + + println!( + " ✅ Epoch {}: W={}, W'(new_reduced)={:?}, W'(super_swiper)={:?}, W'(new_reduced_v2)={:?}", + epoch, + w, + w_prime_new_reduced, + w_prime_super_swiper, + w_prime_new_reduced_v2 + ); + + rows.push(Row { + epoch: *epoch, + w, + w_prime_new_reduced, + w_prime_super_swiper, + w_prime_new_reduced_v2, + }); + } + + let col_w = 8; + let sep = "=".repeat(8 + 3 + col_w + 3 + col_w + 3 + col_w + 3 + col_w); + println!("\n{}", sep); + println!( + "📈 W and W' by algorithm — t={}%·W, L={}%·W, δ_allow={}%·W", + 100.0 * (*params.t_alpha.numer() as f64) / (*params.t_alpha.denom() as f64), + params.liveness_pct * 100.0, + params.allowed_delta_pct * 100.0, + ); + println!("{}", sep); + println!( + "{:<8} | {:| { + x.map(|v| v.to_string()) + .unwrap_or_else(|| "N/A".to_string()) + }; + println!( + "{:<8} | {: L$ then $w'(S) > t' + f'$. + +This is the contrapositive of the right arrow of the $t' + f'$ mapping. + +**3. Byzantine Removal.** If $w'(S) \geq t' + f'$ and $T \subseteq S$ with $w(T) \leq f$, then $w'(S \setminus T) \geq t'$. + +*Proof.* By definition, $f$ is the lower bound of the range that $f'$ maps to, so $w(T) \leq f \implies w'(T) \leq f'$ (left arrow of the $f'$ mapping). Therefore: + +$$ +w'(S \setminus T) = w'(S) - w'(T) \geq (t' + f') - f' = t'. +$$ + +This property ensures that after collecting enough reduced weight ($t' + f'$) in signatures, removing all Byzantine parties (with original weight at most $f$) still leaves at least $t'$ reduced weight — enough to reconstruct. + +## Concrete Comparison + +### Example 1: $t_{min} = 34\%,\ L = 75\%,\ \delta_{allowed} = 8\%$ + +**Unilateral (new_reduced)** with $\delta = 8\%$: + +- $t' = 34\%,\quad f' = 33\%,\quad f = 33\%$ +- $t' = 34\% \to [34\%,\ 42\%]$ +- $f' = 33\% \to [33\%,\ 41\%]$ +- $t' + f' = 67\% \to [67\%,\ 75\%]$ + +**Bilateral (super_swiper)** with $\delta = 4\%$: + +- $t' = 38\%,\quad f' = 33\%,\quad f = 29\%$ +- $t' = 38\% \to [34\%,\ 42\%]$ +- $f' = 33\% \to [29\%,\ 37\%]$ +- $t' + f' = 71\% \to [67\%,\ 75\%]$ + +### Example 2: $t_{min} = 52\%,\ L = 80\%,\ \delta_{allowed} = 8\%$ + +**Unilateral (new_reduced)** with $\delta = 8\%$: + +- $t' = 52\%,\quad f' = 20\%,\quad f = 20\%$ +- $t' = 52\% \to [52\%,\ 60\%]$ +- $f' = 20\% \to [20\%,\ 28\%]$ +- $t' + f' = 72\% \to [72\%,\ 80\%]$ + +**Bilateral (super_swiper)** with $\delta = 4\%$: + +- $t' = 56\%,\quad f' = 20\%,\quad f = 16\%$ +- $t' = 56\% \to [52\%,\ 60\%]$ +- $f' = 20\% \to [16\%,\ 24\%]$ +- $t' + f' = 76\% \to [72\%,\ 80\%]$ diff --git a/fastcrypto-tbls/src/weight_reduction/data/sui_real_all.dat b/fastcrypto-tbls/src/weight_reduction/data/sui_real_all.dat new file mode 100644 index 0000000000..0e1c81b5dd --- /dev/null +++ b/fastcrypto-tbls/src/weight_reduction/data/sui_real_all.dat @@ -0,0 +1,123 @@ +212252538767224829 +188880257735879481 +166026398114086688 +154746213710639463 +150224397567809418 +139551312204498877 +139208548449620023 +135038778696774345 +130979136236399955 +126088636814611458 +124790807181333316 +122672176290093072 +119419199778100533 +117057350535664597 +117050612264325688 +115790432530686217 +115789038946416441 +113440161164268517 +112823132588199183 +110436602539565846 +109238474388271136 +104620119135895572 +103876251733327895 +96055237107199977 +91832312966714024 +86905463516262357 +86170416020656964 +85770815644957987 +85730235547735052 +84915913052209475 +83489532139574661 +83103931009554298 +80482975976267401 +80252450688714802 +79324692744725865 +79319684235804968 +79017608614612474 +77997617898752778 +76767894077397281 +76515355697330367 +75995673208134924 +75480332622165851 +74203303820770754 +71105005064405336 +67737642455845733 +66600812175911146 +65571944222719375 +65468113585398635 +65316900781525323 +64303563133218123 +63771067046668131 +63033212621023100 +61920647396287067 +57732943810816554 +57628360545441842 +55930684518725267 +55920528771555640 +53287317877410932 +51365982252969892 +45255018532578069 +44389878907541404 +44185402890448526 +41365238994036777 +40884326441719853 +39594830164895040 +39495699722274034 +38578900550924908 +38225143411235595 +37822049395101280 +37746971301972396 +37622165783717231 +37603159907115999 +37068436323006525 +35499789531192996 +35295601116898611 +35278699152506204 +35209349470879553 +33784912554732963 +33393377052498882 +32738359679924671 +32665564802263520 +32566675763692689 +32484739644220745 +32245536031549549 +31398934348616554 +31137076349492609 +30911400239794661 +30454333015409524 +30038558801384893 +29881829114792000 +29880238253527729 +29137946735085844 +29066624792355033 +29028465912493177 +29009808831435063 +28994266228046995 +28806361327341044 +28790674402248071 +28261702619534386 +27906077008873101 +27540685934512001 +27138796547792501 +27070348026867013 +26020519298582214 +25789065722341874 +25583164176922702 +25431347792175887 +24879101625521459 +24535593692057345 +24269655070838595 +23678862766749455 +23547854672745594 +22591087890281450 +21694351121478735 +17220631821936868 +16419713410923314 +16037119179243976 +15362702195496342 +15183314763514495 +14085592618374688 +13696803745481288 +3237692406412718 +3001576799280982 diff --git a/fastcrypto-tbls/src/weight_reduction/data/sui_real_all_voting_power_epoch_100.dat b/fastcrypto-tbls/src/weight_reduction/data/sui_real_all_voting_power_epoch_100.dat new file mode 100644 index 0000000000..4387dd21d1 --- /dev/null +++ b/fastcrypto-tbls/src/weight_reduction/data/sui_real_all_voting_power_epoch_100.dat @@ -0,0 +1,105 @@ +391 +325 +286 +275 +255 +238 +235 +206 +206 +198 +193 +192 +192 +190 +188 +186 +186 +185 +185 +185 +175 +169 +158 +158 +158 +158 +158 +151 +151 +146 +145 +144 +118 +110 +98 +94 +89 +84 +76 +71 +71 +64 +63 +62 +62 +60 +59 +57 +57 +57 +54 +50 +50 +50 +49 +48 +48 +48 +48 +47 +47 +47 +47 +47 +47 +47 +47 +47 +47 +47 +47 +47 +45 +41 +40 +38 +38 +37 +37 +37 +37 +37 +37 +37 +37 +37 +37 +37 +37 +37 +37 +37 +37 +36 +36 +36 +36 +36 +36 +36 +36 +36 +36 +35 +35 diff --git a/fastcrypto-tbls/src/weight_reduction/data/sui_real_all_voting_power_epoch_200.dat b/fastcrypto-tbls/src/weight_reduction/data/sui_real_all_voting_power_epoch_200.dat new file mode 100644 index 0000000000..6e638cc2f6 --- /dev/null +++ b/fastcrypto-tbls/src/weight_reduction/data/sui_real_all_voting_power_epoch_200.dat @@ -0,0 +1,105 @@ +335 +288 +285 +279 +261 +243 +231 +227 +216 +200 +196 +189 +188 +187 +179 +176 +175 +175 +174 +171 +169 +169 +169 +169 +168 +168 +160 +144 +144 +144 +144 +144 +137 +137 +131 +131 +131 +131 +109 +83 +80 +72 +68 +68 +66 +64 +61 +59 +56 +55 +52 +51 +51 +49 +45 +45 +43 +43 +43 +43 +43 +43 +43 +43 +43 +43 +43 +43 +43 +43 +43 +43 +41 +39 +37 +37 +34 +34 +34 +34 +33 +33 +33 +33 +33 +33 +33 +33 +33 +33 +33 +33 +33 +33 +33 +33 +33 +33 +33 +33 +33 +33 +33 +32 +32 diff --git a/fastcrypto-tbls/src/weight_reduction/data/sui_real_all_voting_power_epoch_400.dat b/fastcrypto-tbls/src/weight_reduction/data/sui_real_all_voting_power_epoch_400.dat new file mode 100644 index 0000000000..75b0ceb3e5 --- /dev/null +++ b/fastcrypto-tbls/src/weight_reduction/data/sui_real_all_voting_power_epoch_400.dat @@ -0,0 +1,106 @@ +298 +295 +293 +277 +258 +258 +229 +209 +206 +179 +178 +176 +169 +169 +160 +152 +151 +151 +150 +150 +150 +147 +147 +140 +137 +137 +137 +131 +125 +116 +116 +113 +109 +108 +108 +108 +106 +105 +103 +103 +102 +102 +101 +100 +99 +95 +93 +91 +88 +88 +82 +77 +77 +76 +76 +75 +62 +56 +55 +53 +50 +49 +48 +46 +46 +46 +45 +44 +44 +44 +44 +44 +44 +44 +44 +44 +44 +44 +41 +41 +40 +38 +37 +37 +37 +36 +35 +35 +34 +34 +34 +34 +34 +34 +34 +34 +34 +34 +34 +34 +34 +34 +34 +34 +32 +31 diff --git a/fastcrypto-tbls/src/weight_reduction/data/sui_real_all_voting_power_epoch_800.dat b/fastcrypto-tbls/src/weight_reduction/data/sui_real_all_voting_power_epoch_800.dat new file mode 100644 index 0000000000..9986aa1d66 --- /dev/null +++ b/fastcrypto-tbls/src/weight_reduction/data/sui_real_all_voting_power_epoch_800.dat @@ -0,0 +1,116 @@ +285 +278 +199 +192 +189 +181 +178 +177 +173 +167 +163 +159 +158 +155 +154 +154 +153 +153 +149 +146 +145 +140 +139 +138 +135 +130 +123 +120 +115 +115 +114 +114 +114 +111 +110 +107 +106 +106 +105 +104 +103 +102 +100 +98 +98 +89 +88 +87 +87 +85 +85 +84 +83 +78 +77 +72 +69 +68 +61 +61 +59 +59 +58 +53 +52 +51 +51 +50 +50 +50 +50 +50 +49 +49 +49 +49 +48 +48 +47 +46 +45 +45 +45 +44 +44 +42 +42 +42 +42 +42 +41 +40 +40 +40 +39 +39 +39 +38 +38 +38 +38 +38 +38 +38 +37 +36 +36 +36 +35 +32 +31 +31 +29 +28 +26 +19 diff --git a/fastcrypto-tbls/src/weight_reduction/data/sui_real_all_voting_power_epoch_974.dat b/fastcrypto-tbls/src/weight_reduction/data/sui_real_all_voting_power_epoch_974.dat new file mode 100644 index 0000000000..fd55be5b38 --- /dev/null +++ b/fastcrypto-tbls/src/weight_reduction/data/sui_real_all_voting_power_epoch_974.dat @@ -0,0 +1,127 @@ +282 +253 +217 +203 +201 +183 +181 +176 +176 +169 +165 +160 +157 +156 +156 +151 +149 +149 +148 +148 +147 +140 +139 +124 +122 +117 +115 +115 +112 +112 +108 +108 +107 +106 +105 +105 +103 +102 +100 +99 +98 +94 +92 +90 +88 +88 +88 +87 +86 +85 +84 +83 +82 +77 +75 +75 +75 +72 +69 +60 +60 +60 +59 +55 +53 +52 +52 +51 +51 +51 +51 +51 +48 +48 +47 +47 +47 +47 +45 +44 +44 +43 +43 +42 +41 +41 +41 +40 +40 +39 +39 +39 +39 +38 +38 +38 +38 +38 +38 +37 +37 +36 +36 +35 +35 +34 +34 +34 +33 +32 +31 +31 +30 +29 +26 +25 +22 +21 +21 +20 +20 +18 +13 +7 +4 +4 +3 diff --git a/fastcrypto-tbls/src/weight_reduction/mod.rs b/fastcrypto-tbls/src/weight_reduction/mod.rs new file mode 100644 index 0000000000..b1a3367d12 --- /dev/null +++ b/fastcrypto-tbls/src/weight_reduction/mod.rs @@ -0,0 +1,557 @@ +// Copyright (c) 2022, Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//! The implementation of the algorithms presented in the paper +//! Weight reduction in distributed protocols: new algorithms and analysis +//! [paper](https://eprint.iacr.org/2025/1076). +//! Adapted from: https://github.com/tolikzinovyev/weight-reduction + +use fastcrypto::error::{FastCryptoError, FastCryptoResult}; +use itertools::Itertools; +use std::cmp::Ordering; +use std::collections::{BTreeSet, BinaryHeap}; +use std::ops::Index; + +pub mod weight_reduction_checks; + +// Type alias for rational numbers used in weight reduction +pub type Ratio = num_rational::Ratio; + +// Helper functions for weight reduction calculations +fn max_adv_weight(alpha: Ratio, total_weight: u64) -> u64 { + (alpha * total_weight).to_integer() +} + +fn max_adv_weight_from_weights(alpha: Ratio, weights: &[u64]) -> u64 { + max_adv_weight(alpha, weights.iter().sum()) +} + +fn adv_tickets_target(beta: Ratio, total_num_tickets: u64) -> u64 { + (beta * total_num_tickets).ceil().to_integer() +} + +// Dynamic Programming data structure for knapsack calculations +#[derive(Debug)] +struct DP { + max_weight: u64, + adv_tickets_target: u64, + dp: Vec, +} + +impl DP { + /// Create a knapsack dynamic programming data with configured max weight + /// and adversarial tickets target. Returns None only when it's immediately + /// clear we can achieve the adversarial tickets target -- if and only if + /// adv_tickets_target = 0. + fn new(max_weight: u64, adv_tickets_target: u64) -> Option { + if adv_tickets_target == 0 { + return None; + } + Some(DP { + max_weight, + adv_tickets_target, + dp: vec![0], + }) + } + + /// Create a copy of the data structure with a new configured adversarial + /// tickets target. It must be less or equal to the previously configured + /// adversarial tickets target. Returns None iff the new adversarial tickets + /// target has already been achieved. + fn make_copy(&self, adv_tickets_target: u64) -> Option { + assert!(adv_tickets_target <= self.adv_tickets_target); + + if self.dp.len() > adv_tickets_target as usize { + return None; + } + + Some(DP { + max_weight: self.max_weight, + adv_tickets_target, + dp: self.dp.clone(), + }) + } + + /// Apply an element with weight w and t tickets. Returns None iff + /// the configured adversarial tickets target is achieved. + fn apply(mut self, w: u64, t: u64) -> Option { + assert!(w > 0); + + if w > self.max_weight || t == 0 { + return Some(self); + } + if t >= self.adv_tickets_target { + return None; + } + + for i in (1..self.dp.len()).rev() { + if self.dp[i] != 0 { + let accumulated_weight = self.dp[i] + w; + let accumulated_tickets = i + t as usize; + if accumulated_weight <= self.max_weight { + if accumulated_tickets >= self.adv_tickets_target as usize { + return None; + } + ensure_size(&mut self.dp, accumulated_tickets + 1); + if accumulated_weight < self.dp[accumulated_tickets] + || self.dp[accumulated_tickets] == 0 + { + self.dp[accumulated_tickets] = accumulated_weight; + } + } + } + } + + let t = t as usize; + ensure_size(&mut self.dp, t + 1); + if w < self.dp[t] || self.dp[t] == 0 { + self.dp[t] = w; + } + + Some(self) + } + + /// Returns the maximum achievable adversarial number of tickets. + #[cfg(test)] + fn adversarial_tickets(&self) -> u64 { + self.dp + .iter() + .rposition(|&w| w != 0) + .map(|t| t as u64) + .unwrap_or(0) + } +} + +// Tickets data structure for managing ticket assignments +#[derive(Debug, Clone)] +struct Tickets { + tickets: Vec, + total: u64, +} + +impl Tickets { + fn new() -> Self { + Self { + tickets: Vec::new(), + total: 0, + } + } + + fn update(&mut self, index: usize) { + ensure_size(&mut self.tickets, index + 1); + self.tickets[index] += 1; + self.total += 1; + } + + fn into_vec(self) -> Vec { + self.tickets + } + + fn update_many(&mut self, indices: &[usize]) { + for &index in indices { + self.update(index); + } + } + + #[cfg(test)] + fn from_vec(tickets: Vec) -> Self { + let total = tickets.iter().sum(); + Self { tickets, total } + } +} + +impl Index for Tickets { + type Output = u64; + fn index(&self, index: usize) -> &u64 { + self.tickets.get(index).unwrap_or(&0) + } +} + +// Helper types for generating deltas +#[derive(Eq, PartialEq)] +struct QueueElement { + s: Ratio, + i: usize, +} + +impl Ord for QueueElement { + fn cmp(&self, other: &Self) -> Ordering { + other.s.cmp(&self.s).then(other.i.cmp(&self.i)) + } +} + +impl PartialOrd for QueueElement { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +struct Generator<'a> { + weights: &'a [u64], + c: Ratio, + r: usize, + queue: BinaryHeap, +} + +impl<'a> Generator<'a> { + fn new(weights: &'a [u64], c: Ratio) -> FastCryptoResult { + debug_assert!(weights.windows(2).all(|w| w[0] >= w[1])); + if weights.is_empty() + || weights.last().copied().unwrap_or(0) == 0 + || c < 0.into() + || c >= 1.into() + { + return Err(FastCryptoError::InvalidInput); + } + let queue = BinaryHeap::from([QueueElement { + s: (Ratio::from_integer(1) - c) / Ratio::from_integer(*weights.first().unwrap()), + i: 0, + }]); + + Ok(Self { + weights, + c, + r: 0, + queue, + }) + } +} + +impl Iterator for Generator<'_> { + type Item = usize; + + fn next(&mut self) -> Option { + let QueueElement { s, i } = self.queue.pop()?; + let new_value = (s * self.weights[i] + self.c).to_integer(); + + self.queue.push(QueueElement { + s: (Ratio::from_integer(new_value + 1) - self.c) / Ratio::from_integer(self.weights[i]), + i, + }); + if i == self.r && self.r + 1 < self.weights.len() { + self.r += 1; + self.queue.push(QueueElement { + s: (Ratio::from_integer(1) - self.c) / Ratio::from_integer(self.weights[self.r]), + i: self.r, + }); + } + + Some(i) + } +} + +fn generate_deltas(weights: &[u64], c: Ratio) -> impl Iterator + '_ { + Generator::new(weights, c).expect("Invalid input to generate_deltas") +} + +/// Calculates the head indices for the current batch. +fn indices_head(tickets_len: usize, deltas: &[usize]) -> impl Iterator + '_ { + set_minus(0..tickets_len, deltas.iter()) +} + +/// Calculates the DP data structure with indices applied that are not in `delta`s. +/// Returns None iff the DP head cannot be constructed which means +/// that the current batch should be skipped. +fn dp_head( + beta: Ratio, + weights: &[u64], + max_adv_weight: u64, + deltas: &[usize], + tickets: &Tickets, +) -> Option { + indices_head(tickets.tickets.len(), deltas).try_fold( + DP::new( + max_adv_weight, + adv_tickets_target(beta, tickets.total + deltas.len() as u64), + )?, + |dp, index| dp.apply(weights[index], tickets[index]), + ) +} + +/// Apply those indices to dp_head that are in `add_indices` but not in +/// `exclude_indices`. +fn apply( + weights: &[u64], + dp_head: &DP, + tickets: &Tickets, + adv_tickets_target: u64, + add_indices: &[usize], + exclude_indices: &[usize], +) -> Option { + set_minus(add_indices.iter().copied(), exclude_indices.iter()) + .try_fold(dp_head.make_copy(adv_tickets_target)?, |dp, i| { + dp.apply(weights[i], tickets[i]) + }) +} + +/// Apply `deltas` to provided `tickets`. If after applying 0 or more +/// deltas, a valid ticket assignment is found, returns true with +/// `tickets` containing the corresponding ticket assignment. +/// Otherwise, returns false with `tickets` containing +/// the new ticket assignment after applying all deltas. +/// `dp_head` must have all indices applied except those in `deltas` with data +/// in `tickets`. +/// +/// # Recursion Depth Bound +/// The recursion depth is bounded by O(log n) where n = `deltas.len()`, because: +/// - Each recursive call splits `deltas` roughly in half +/// - The base case is when `deltas.is_empty()` +/// - Maximum depth is approximately log₂(n) + 1 +/// +/// Since `deltas` comes from `batch_size - 1` in `solve()`, and `batch_size` grows +/// exponentially, the recursion depth grows logarithmically. +fn process_batch_recursive( + beta: Ratio, + weights: &[u64], + deltas: &[usize], + dp_head: &DP, + tickets: &mut Tickets, +) -> bool { + if deltas.is_empty() { + // All indices in `deltas` were successfully applied without the adversary + // winning. We found a solution. + // Sanity check: `dp_head` must have the right target. + assert_eq!( + dp_head.adv_tickets_target, + adv_tickets_target(beta, tickets.total) + ); + return true; + } + + // Number of tickets assignments in the left branch. + let left_branch_size = deltas.len().div_ceil(2); + + let deltas_left = &deltas[..left_branch_size - 1]; + if let Some(dp) = apply( + weights, + dp_head, + tickets, + adv_tickets_target(beta, tickets.total + deltas_left.len() as u64), + deltas, + deltas_left, + ) { + if process_batch_recursive(beta, weights, deltas_left, &dp, tickets) { + return true; + } + } else { + // Apply the left deltas before continuing. + tickets.update_many(deltas_left); + } + tickets.update(deltas[left_branch_size - 1]); + + let deltas_right = &deltas[left_branch_size..]; + if let Some(dp) = apply( + weights, + dp_head, + tickets, + adv_tickets_target(beta, tickets.total + deltas_right.len() as u64), + deltas, + deltas_right, + ) { + process_batch_recursive(beta, weights, deltas_right, &dp, tickets) + } else { + // Apply the rest of the deltas before exiting. + tickets.update_many(deltas_right); + false + } +} + +/// Apply `deltas` to provided `tickets`. If after applying 0 or more +/// deltas a valid ticket assignment is found, returns true with +/// `tickets` containing the corresponding ticket assignment. +/// Otherwise, returns false with `tickets` containing +/// the new ticket assignment after applying all deltas. +fn process_batch( + beta: Ratio, + weights: &[u64], + max_adv_weight: u64, + deltas: &[usize], + tickets: &mut Tickets, +) -> bool { + let Some(dp_head) = dp_head(beta, weights, max_adv_weight, deltas, tickets) else { + // We are exiting early. Apply all of the deltas before that. + tickets.update_many(deltas); + return false; + }; + process_batch_recursive(beta, weights, deltas, &dp_head, tickets) +} + +pub fn solve(alpha: Ratio, beta: Ratio, weights: &[u64]) -> Vec { + debug_assert!(weights.windows(2).all(|w| w[0] >= w[1])); + + let max_adv_weight = max_adv_weight_from_weights(alpha, weights); + + let mut tickets = Tickets::new(); + let mut g = generate_deltas(weights, alpha); + + let mut batch_size: usize = 1; + // This loop terminates because: + // 1. The generator `g` is infinite (always produces deltas via the queue-based algorithm) + // 2. We use exponential backoff: batch_size doubles each iteration (1, 2, 4, 8, ...) + // 3. The algorithm is guaranteed to find a solution (by the paper's theoretical results) + // 4. Once batch_size is large enough to include all deltas needed for a valid solution, + // `process_batch_recursive` will eventually return true when it successfully applies + // all deltas in the batch without the adversary winning (see the empty deltas base case) + // + // # Overflow Safety + // The paper guarantees a solution with at most O(n) total tickets, where n = weights.len(). + // Since we need at most n indices (one per weight) to form a solution, `batch_size` will + // never need to exceed n. We cap it at `weights.len()` to prevent overflow and ensure + // termination even in edge cases. + let max_batch_size = weights.len(); + loop { + tickets.update(g.next().unwrap()); + let deltas = (&mut g).take(batch_size - 1).collect_vec(); + + if process_batch(beta, weights, max_adv_weight, &deltas, &mut tickets) { + return tickets.into_vec(); + } + + // Prevent overflow: cap batch_size at max_batch_size + batch_size = (batch_size * 2).min(max_batch_size); + } +} + +/// If `vector` is smaller than `size`, append default values to match size. +/// Otherwise, do nothing. +fn ensure_size(vector: &mut Vec, size: usize) { + if vector.len() < size { + vector.resize(size, T::default()); + } +} + +/// Return all elements from `base` that is not in `to_exclude` in O(max(n log n, |base|)) time where `n = |to_exclude|`. +fn set_minus<'a, T: Ord + 'a>( + base: impl Iterator + 'a, + to_exclude: impl Iterator, +) -> impl Iterator + 'a { + let excluded = to_exclude.into_iter().collect::>(); + base.filter(move |i| !excluded.contains(i)) +} + +#[cfg(test)] +mod calc_indices_head_tail_tests { + use super::indices_head; + use itertools::Itertools; + use test_case::test_case; + + struct TestCase<'a> { + tickets_len: usize, + deltas: &'a [usize], + expected: Vec, + } + + #[test_case( + TestCase { + tickets_len: 0, + deltas: &[0, 1], + expected: vec![], + }; + "zero_tickets" + )] + #[test_case( + TestCase { + tickets_len: 5, + deltas: &[1, 3], + expected: vec![0, 2, 4], + }; + "multiple_tickets" + )] + #[test_case( + TestCase { + tickets_len: 5, + deltas: &[3, 3], + expected: vec![0, 1, 2, 4], + }; + "index_updated_multiple_times" + )] + #[test_case( + TestCase { + tickets_len: 5, + deltas: &[0, 4], + expected: vec![1, 2, 3], + }; + "first_last_index_updated" + )] + fn all(mut test_case: TestCase<'_>) { + let mut ret = indices_head(test_case.tickets_len, test_case.deltas).collect_vec(); + test_case.expected.sort_unstable(); + ret.sort_unstable(); + assert_eq!(test_case.expected, ret); + } +} + +#[cfg(test)] +mod calc_dp_head_tests { + use super::{dp_head, solve, Ratio, Tickets, DP}; + + fn calc_dp_head_helper( + beta: Ratio, + weights: &[u64], + max_adv_weight: u64, + deltas: &[usize], + tickets: &[u64], + ) -> Option { + let tickets = Tickets::from_vec(tickets.to_vec()); + dp_head(beta, weights, max_adv_weight, deltas, &tickets) + } + + #[test] + fn zero_tickets() { + let beta = Ratio::new(1, 2); + let weights = &[20, 30]; + let max_adv_weight = 50; + let deltas = &[0, 1]; + let tickets = &[]; + + let dp = calc_dp_head_helper(beta, weights, max_adv_weight, deltas, tickets).unwrap(); + + assert_eq!(0, dp.adversarial_tickets()); + } + + #[test] + fn many_adversarial_tickets() { + let beta = Ratio::new(1, 2); + let weights = &[20, 30]; + let max_adv_weight = 50; + let deltas = &[1, 2]; + let tickets = &[3]; + + assert!(calc_dp_head_helper(beta, weights, max_adv_weight, deltas, tickets).is_none()); + } + + #[test] + fn basic() { + let beta = Ratio::new(1, 2); + let weights = &[20, 30, 10, 10]; + let max_adv_weight = 50; + let deltas = &[2, 5]; + let tickets = &[2, 3, 9, 1]; + + let dp = calc_dp_head_helper(beta, weights, max_adv_weight, deltas, tickets).unwrap(); + + assert_eq!(5, dp.adversarial_tickets()); + } + + #[test] + fn test_reduction() { + let alpha = Ratio::new(1, 3); + let beta = Ratio::new(2, 5); + let weights = &[ + 391, 325, 286, 275, 255, 238, 235, 206, 206, 198, 193, 192, 192, 190, 188, 186, 186, + 185, 185, 185, 175, 169, 158, 158, 158, 158, 158, 151, 151, 146, 145, 144, 118, 110, + 98, 94, 89, 84, 76, 71, 71, 64, 63, 62, 62, 60, 59, 57, 57, 57, 54, 50, 50, 50, 49, 48, + 48, 48, 48, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 45, 41, 40, 38, 38, 37, + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 36, 36, 36, 36, 36, 36, 36, + 36, 36, 36, 35, 35, + ]; + let new_weights = solve(alpha, beta, weights); + let expected = vec![ + 7, 6, 5, 5, 5, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + ]; + assert_eq!(expected, new_weights); + } +} diff --git a/fastcrypto-tbls/src/weight_reduction/scripts/fetch_all_sui_validators.py b/fastcrypto-tbls/src/weight_reduction/scripts/fetch_all_sui_validators.py new file mode 100644 index 0000000000..3f53a362ca --- /dev/null +++ b/fastcrypto-tbls/src/weight_reduction/scripts/fetch_all_sui_validators.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +""" +Script to fetch ALL Sui validators using GraphQL pagination. +""" + +import json +import subprocess +import sys +from typing import List, Tuple, Optional + +def fetch_all_validators_with_pagination(epoch_id: int = 930) -> List[Tuple[str, int]]: + """ + Fetch all validators using GraphQL pagination. + """ + all_validators = [] + has_next_page = True + after_cursor = None + + print(f"🔍 Fetching all validators for epoch {epoch_id}...") + + while has_next_page: + # Build the query with pagination + if after_cursor: + query = f""" + {{ + epoch(epochId: {epoch_id}) {{ + validatorSet {{ + activeValidators(after: "{after_cursor}") {{ + pageInfo {{ + hasNextPage + endCursor + }} + nodes {{ + name + nextEpochStake + }} + }} + }} + }} + }} + """ + else: + query = f""" + {{ + epoch(epochId: {epoch_id}) {{ + validatorSet {{ + activeValidators {{ + pageInfo {{ + hasNextPage + endCursor + }} + nodes {{ + name + nextEpochStake + }} + }} + }} + }} + }} + """ + + print(f"📡 Fetching page (after: {after_cursor or 'start'})...") + + try: + payload = { + "query": query, + "variables": {} + } + + result = subprocess.run([ + 'curl', '-s', '-X', 'POST', + '-H', 'Content-Type: application/json', + '-H', 'Accept: application/json', + 'https://graphql.mainnet.sui.io/graphql', + '-d', json.dumps(payload) + ], capture_output=True, text=True, timeout=30) + + if result.returncode != 0: + print(f"❌ Error making request: {result.stderr}") + break + + response = json.loads(result.stdout) + + if "errors" in response: + print(f"❌ GraphQL errors: {response['errors']}") + break + + if "data" not in response: + print("❌ No data in response") + break + + # Extract validators from this page + epoch = response["data"]["epoch"] + validator_set = epoch["validatorSet"] + active_validators = validator_set["activeValidators"] + page_info = active_validators["pageInfo"] + nodes = active_validators["nodes"] + + print(f" 📊 Found {len(nodes)} validators on this page") + + # Process validators + for validator in nodes: + name = validator.get("name", "") + stake = validator.get("nextEpochStake", 0) + + if stake: + try: + stake_int = int(stake) + all_validators.append((name, stake_int)) + print(f" ✅ {name}: {stake_int:,} SUI") + except ValueError: + print(f" ⚠️ Could not parse stake for {name}: {stake}") + + # Update pagination info + has_next_page = page_info.get("hasNextPage", False) + after_cursor = page_info.get("endCursor") + + print(f" 📄 Has next page: {has_next_page}") + if after_cursor: + print(f" 🔄 Next cursor: {after_cursor}") + + except Exception as e: + print(f"❌ Error fetching page: {e}") + break + + print(f"\n🎉 Total validators fetched: {len(all_validators)}") + return all_validators + +def save_to_dat_file(validators: List[Tuple[str, int]], filename: str) -> None: + """Save validator stakes to a .dat file in descending order.""" + if not validators: + print("No validator data to save.") + return + + # Sort by stake amount in descending order + validators.sort(key=lambda x: x[1], reverse=True) + + # Extract just the stake amounts + stakes = [stake for _, stake in validators] + + with open(filename, 'w') as f: + for stake in stakes: + f.write(f"{stake}\n") + + print(f"✅ Saved {len(stakes)} validator stake weights to {filename}") + print(f"Total stake: {sum(stakes):,}") + print(f"Largest stake: {stakes[0]:,}") + print(f"Smallest stake: {stakes[-1]:,}") + + # Also save detailed info + detail_filename = filename.replace('.dat', '_details.txt') + with open(detail_filename, 'w') as f: + f.write("Validator Name,Stake Amount\n") + for name, stake in validators: + f.write(f"{name},{stake}\n") + + print(f"Detailed validator info saved to {detail_filename}") + +def test_with_algorithms(filename: str) -> None: + """Test the generated data with weight reduction algorithms.""" + try: + print(f"\n🧪 Testing {filename} with weight reduction algorithms...") + + result = subprocess.run([ + 'cargo', 'run', '--bin', 'solve', '--', + '--algorithm', 'faster-swiper', + '--alpha', '1/5', + '--beta', '1/3', + '--weights-path', filename, + '--show-tickets' + ], capture_output=True, text=True, timeout=30) + + if result.returncode == 0: + print("✅ Successfully tested with weight reduction algorithms!") + print(result.stdout) + else: + print("❌ Error testing with algorithms:") + print(result.stderr) + + except Exception as e: + print(f"❌ Error running algorithms: {e}") + +def main(): + """Main function to fetch all Sui validators.""" + print("🚀 Fetching ALL Sui validators using GraphQL pagination...") + + # Get current epoch + try: + result = subprocess.run([ + 'curl', '-s', '-X', 'POST', + '-H', 'Content-Type: application/json', + 'https://graphql.mainnet.sui.io/graphql', + '-d', '{"query": "{ epoch { epochId } }"}' + ], capture_output=True, text=True, timeout=15) + + if result.returncode == 0: + response = json.loads(result.stdout) + current_epoch = response["data"]["epoch"]["epochId"] + print(f"📅 Current epoch: {current_epoch}") + else: + current_epoch = 930 + print(f"⚠️ Using default epoch: {current_epoch}") + except: + current_epoch = 930 + print(f"⚠️ Using default epoch: {current_epoch}") + + # Fetch all validators + validators = fetch_all_validators_with_pagination(current_epoch) + + if validators: + # Save to .dat file + output_file = "data/sui_real_all.dat" + save_to_dat_file(validators, output_file) + + # Test with algorithms + test_with_algorithms(output_file) + + print(f"\n🎉 Successfully created complete Sui validator data!") + print(f"📁 Main file: {output_file}") + print(f"📁 Details file: {output_file.replace('.dat', '_details.txt')}") + + else: + print("\n❌ Could not fetch validator data.") + +if __name__ == "__main__": + main() diff --git a/fastcrypto-tbls/src/weight_reduction/scripts/fetch_all_sui_validators_voting_power.py b/fastcrypto-tbls/src/weight_reduction/scripts/fetch_all_sui_validators_voting_power.py new file mode 100644 index 0000000000..d78e7fb0e7 --- /dev/null +++ b/fastcrypto-tbls/src/weight_reduction/scripts/fetch_all_sui_validators_voting_power.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +""" +Script to fetch ALL Sui validators' voting power using GraphQL pagination. +""" + +import json +import subprocess +from typing import List, Tuple + +def fetch_all_validators_with_pagination(epoch_id: int = 930) -> List[Tuple[str, int]]: + """ + Fetch all validators' voting power using GraphQL pagination. + """ + all_validators = [] + has_next_page = True + after_cursor = None + + print(f"🔍 Fetching all validators' voting power for epoch {epoch_id}...") + + while has_next_page: + # Build the query with pagination + if after_cursor: + query = f""" + {{ + epoch(epochId: {epoch_id}) {{ + validatorSet {{ + activeValidators(after: "{after_cursor}") {{ + pageInfo {{ + hasNextPage + endCursor + }} + nodes {{ + name + votingPower + }} + }} + }} + }} + }} + """ + else: + query = f""" + {{ + epoch(epochId: {epoch_id}) {{ + validatorSet {{ + activeValidators {{ + pageInfo {{ + hasNextPage + endCursor + }} + nodes {{ + name + votingPower + }} + }} + }} + }} + }} + """ + + print(f"📡 Fetching page (after: {after_cursor or 'start'})...") + + try: + payload = { + "query": query, + "variables": {} + } + + result = subprocess.run([ + 'curl', '-s', '-X', 'POST', + '-H', 'Content-Type: application/json', + '-H', 'Accept: application/json', + 'https://graphql.mainnet.sui.io/graphql', + '-d', json.dumps(payload) + ], capture_output=True, text=True, timeout=30) + + if result.returncode != 0: + print(f"❌ Error making request: {result.stderr}") + break + + response = json.loads(result.stdout) + + if "errors" in response: + print(f"❌ GraphQL errors: {response['errors']}") + break + + if "data" not in response: + print("❌ No data in response") + break + + # Extract validators from this page + epoch = response["data"]["epoch"] + validator_set = epoch["validatorSet"] + active_validators = validator_set["activeValidators"] + page_info = active_validators["pageInfo"] + nodes = active_validators["nodes"] + + print(f" 📊 Found {len(nodes)} validators on this page") + + # Process validators + for validator in nodes: + name = validator.get("name", "") + voting_power = validator.get("votingPower", 0) + + if voting_power: + try: + voting_power_int = int(voting_power) + all_validators.append((name, voting_power_int)) + print(f" ✅ {name}: {voting_power_int:,} voting power") + except ValueError: + print(f" ⚠️ Could not parse voting power for {name}: {voting_power}") + else: + print(f" ⚠️ No voting power data for {name}") + + # Update pagination info + has_next_page = page_info.get("hasNextPage", False) + after_cursor = page_info.get("endCursor") + + print(f" 📄 Has next page: {has_next_page}") + if after_cursor: + print(f" 🔄 Next cursor: {after_cursor}") + + except Exception as e: + print(f"❌ Error fetching page: {e}") + break + + print(f"\n🎉 Total validators fetched: {len(all_validators)}") + return all_validators + +def save_to_dat_file(validators: List[Tuple[str, int]], filename: str) -> None: + """Save validator voting power to a .dat file in descending order.""" + if not validators: + print("No validator data to save.") + return + + # Sort by voting power amount in descending order + validators.sort(key=lambda x: x[1], reverse=True) + + # Extract just the voting power amounts + voting_powers = [voting_power for _, voting_power in validators] + + with open(filename, 'w') as f: + for voting_power in voting_powers: + f.write(f"{voting_power}\n") + + print(f"✅ Saved {len(voting_powers)} validator voting power weights to {filename}") + print(f"Total voting power: {sum(voting_powers):,}") + print(f"Largest voting power: {voting_powers[0]:,}") + print(f"Smallest voting power: {voting_powers[-1]:,}") + + # Also save detailed info + detail_filename = filename.replace('.dat', '_details.txt') + with open(detail_filename, 'w') as f: + f.write("Validator Name,Voting Power\n") + for name, voting_power in validators: + f.write(f"{name},{voting_power}\n") + + print(f"Detailed validator info saved to {detail_filename}") + +def main(): + """Main function to fetch all Sui validators' voting power.""" + import sys + + # Get epoch from command line argument or use current epoch + if len(sys.argv) > 1: + try: + epoch = int(sys.argv[1]) + print(f"🚀 Fetching Sui validators' voting power for epoch {epoch}...") + except ValueError: + print(f"❌ Invalid epoch: {sys.argv[1]}. Using current epoch instead.") + epoch = None + else: + epoch = None + + if epoch is None: + print("🚀 Fetching ALL Sui validators' voting power using GraphQL pagination...") + # Get current epoch + try: + result = subprocess.run([ + 'curl', '-s', '-X', 'POST', + '-H', 'Content-Type: application/json', + 'https://graphql.mainnet.sui.io/graphql', + '-d', '{"query": "{ epoch { epochId } }"}' + ], capture_output=True, text=True, timeout=15) + + if result.returncode == 0: + response = json.loads(result.stdout) + epoch = response["data"]["epoch"]["epochId"] + print(f"📅 Current epoch: {epoch}") + else: + epoch = 930 + print(f"⚠️ Using default epoch: {epoch}") + except: + epoch = 930 + print(f"⚠️ Using default epoch: {epoch}") + + # Fetch all validators + validators = fetch_all_validators_with_pagination(epoch) + + if validators: + # Save to .dat file with epoch in filename + if len(sys.argv) > 1: + output_file = f"data/sui_real_all_voting_power_epoch_{epoch}.dat" + else: + output_file = "data/sui_real_all_voting_power.dat" + save_to_dat_file(validators, output_file) + + print(f"\n🎉 Successfully created complete Sui validator voting power data!") + print(f"📁 Main file: {output_file}") + print(f"📁 Details file: {output_file.replace('.dat', '_details.txt')}") + + else: + print("\n❌ Could not fetch validator voting power data.") + +if __name__ == "__main__": + main() + diff --git a/fastcrypto-tbls/src/weight_reduction/weight_reduction_checks.rs b/fastcrypto-tbls/src/weight_reduction/weight_reduction_checks.rs new file mode 100644 index 0000000000..b9c8870ad3 --- /dev/null +++ b/fastcrypto-tbls/src/weight_reduction/weight_reduction_checks.rs @@ -0,0 +1,184 @@ +// Copyright (c) 2022, Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//! Weight reduction checks. See `weight_reduction/PROOF.md` for definitions: +//! - Original weights w_i, reduced weights w'_i, total W = Σw_i, W' = Σw'_i. +//! - Effective divisor d = W/W' (exact ratio). +//! - Precision loss δ = Σᵢ max(w_i − w'_i·d, 0). +//! - Key property: for any coalition S, w(S) − w'(S)·d ≤ δ. + +use num_rational::Ratio; +use rand::Rng; + +/// Calculate the maximum delta across top, bottom, and random validity checks. +/// +/// # Parameters +/// - `t_prime`: Threshold in reduced weights (beta * total_new_weights) +/// - `old_weights`: Original weights (in original order) +/// - `reduced_weights`: Reduced weights (in original order) +/// - `t`: Input threshold (alpha * old_weights_total) +/// - `n_random`: Number of random subsets to test +/// +/// # Returns +/// The maximum delta value from all checks, or None if target cannot be reached +/// +/// # Delta Calculation +/// For each validity check (top, bottom, or random): +/// 1. Take reduced weights to reach 2t' - 1 +/// 2. Let w1 = sum of old weights corresponding to the subset +/// 3. Then delta = w1 - (2t - 1), where t is the input threshold +pub fn get_delta( + t_prime: u64, + old_weights: &[u64], + reduced_weights: &[u64], + t: u64, + n_random: usize, +) -> Option { + if old_weights.len() != reduced_weights.len() { + return None; + } + + // Helper function to calculate delta for a given sorted order + let calculate_delta_for_subset = |indexed: &[(usize, u64, u64)]| -> Option { + let reduced_target = t_prime.saturating_mul(2).saturating_sub(1); + let mut reduced_sum = 0u64; + let mut subset_indices = Vec::new(); + + for (idx, _old_w, red_w) in indexed { + if reduced_sum >= reduced_target { + break; + } + reduced_sum += red_w; + subset_indices.push(*idx); + } + + // If we couldn't reach reduced_target, return None + if reduced_sum < reduced_target { + return None; + } + + // Calculate w1 = sum of old weights corresponding to the subset + let w1: u64 = subset_indices.iter().map(|&idx| old_weights[idx]).sum(); + + let t_target = t.saturating_mul(2).saturating_sub(1); + + // Calculate delta = w1 - (2t - 1) + // Skip if delta would be negative (w1 < 2t - 1) + if w1 < t_target { + return None; + } + + let delta = w1 - t_target; + Some(delta) + }; + + // Check top weights (sorted by reduced weight descending) + let mut indexed_top: Vec<(usize, u64, u64)> = old_weights + .iter() + .enumerate() + .zip(reduced_weights.iter()) + .map(|((i, &old_w), &red_w)| (i, old_w, red_w)) + .collect(); + indexed_top.sort_by(|a, b| b.2.cmp(&a.2)); // Sort by reduced weight descending + let delta_top = calculate_delta_for_subset(&indexed_top); + + // Check bottom weights (sorted by reduced weight ascending) + let mut indexed_bot: Vec<(usize, u64, u64)> = old_weights + .iter() + .enumerate() + .zip(reduced_weights.iter()) + .map(|((i, &old_w), &red_w)| (i, old_w, red_w)) + .collect(); + indexed_bot.sort_by(|a, b| a.2.cmp(&b.2)); // Sort by reduced weight ascending + let delta_bot = calculate_delta_for_subset(&indexed_bot); + + // Generate n random subsets + let mut delta_random = Vec::new(); + let mut rng = rand::thread_rng(); + + for _ in 0..n_random { + // Create a random permutation of indices + let mut indexed_random: Vec<(usize, u64, u64)> = old_weights + .iter() + .enumerate() + .zip(reduced_weights.iter()) + .map(|((i, &old_w), &red_w)| (i, old_w, red_w)) + .collect(); + + // Shuffle randomly + for i in 0..indexed_random.len() { + let j = rng.gen_range(i..indexed_random.len()); + indexed_random.swap(i, j); + } + + // Calculate delta for this random ordering + if let Some(delta) = calculate_delta_for_subset(&indexed_random) { + delta_random.push(delta); + } + } + + // If any validity check resulted in a negative delta (None), skip this solution + // Top and bottom checks must both pass + let delta_top_value = delta_top?; // Top check had negative delta, skip solution + let delta_bot_value = delta_bot?; // Bottom check had negative delta, skip solution + + // All random subsets must pass (no negative deltas) + if delta_random.len() < n_random { + // Some random subsets were skipped due to negative delta, skip solution + return None; + } + + // Collect all delta values (all should be valid at this point) + let mut all_deltas = Vec::new(); + all_deltas.push(delta_top_value); + all_deltas.push(delta_bot_value); + all_deltas.extend(delta_random); + + // Return the maximum delta + Some(all_deltas.iter().fold(0u64, |a, &b| a.max(b))) +} + +/// Compute the global precision loss δ for super_swiper weight reduction (PROOF.md). +/// +/// Definitions: +/// - d = W/W' (exact ratio, no floor), where W = total original weight, W' = total reduced weight. +/// - δ = Σᵢ max(w_i − w'_i·d, 0). This is the precision loss used in the liveness condition. +/// +/// Returns (δ, d). +pub fn compute_precision_loss( + original_weights: &[u64], + reduced_weights: &[u64], +) -> (Ratio, Ratio) { + let total_original: u64 = original_weights.iter().sum(); + let total_reduced: u64 = reduced_weights.iter().sum(); + + // d = total_original / total_reduced (exact ratio, no floor) + let d = if total_reduced > 0 { + Ratio::new(total_original, total_reduced) + } else { + Ratio::from_integer(1) + }; + + // delta = sum(max(original[i] - reduced[i] * d, 0)) + let delta: Ratio = original_weights + .iter() + .enumerate() + .map(|(i, &orig)| { + let red = if i < reduced_weights.len() { + reduced_weights[i] + } else { + 0 + }; + let orig_ratio = Ratio::from_integer(orig); + let scaled_red = Ratio::from_integer(red) * d; + // Compute max(original - scaled_reduced, 0) + if orig_ratio >= scaled_red { + orig_ratio - scaled_red + } else { + Ratio::from_integer(0) + } + }) + .sum(); + + (delta, d) +}