diff --git a/app-policy/checker/bench_egress_test.go b/app-policy/checker/bench_egress_test.go index bb37194784f..4a5d3204336 100644 --- a/app-policy/checker/bench_egress_test.go +++ b/app-policy/checker/bench_egress_test.go @@ -15,17 +15,12 @@ package checker // Benchmark for Evaluate() against a single-tier egress allow-list, the second rule-set shape -// measured in production (the first is in bench_test.go). All data is synthetic; the scale -// parameters and distributions below are anonymized measurements from a policy dump. +// measured in production (the first is in bench_test.go). The set is policyscale.EgressAllowList; +// see policyscale.DefaultEgress for the measured distributions it reproduces. // // go test ./app-policy/checker/ -run '^$' -bench BenchmarkEvaluateEgressAllowList \ // -benchmem -benchtime 100x -cpu 1 // -// Shape: one tier, ~301 policies with no selector (so all of them apply to every endpoint), -// ~18,600 egress rules, almost all Pass. The tier is a destination allow-list: match a rule -// and leave the tier, match nothing and the tier default denies. Every rule's source block is -// empty; each matches a destination address plus a handful of destination ports. -// // The three cases below must be read separately, because criterion ordering helps them by // different amounts. A flow whose port is shared by few rules is rejected on the port // comparison almost everywhere and never pays for the address criteria; a flow on a popular @@ -34,66 +29,14 @@ package checker // depth. Averaging the three hides that. import ( - "fmt" - "math/rand" - "net" "testing" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" - "github.com/projectcalico/calico/app-policy/policystore" - "github.com/projectcalico/calico/felix/proto" + "github.com/projectcalico/calico/app-policy/policyscale" "github.com/projectcalico/calico/felix/rules" - "github.com/projectcalico/calico/felix/types" -) - -const ( - egressNumPolicies = 301 - egressRulesPerPolicy = 62 // 18,662 rules. - - // Fraction of rules whose destination is a selector, which reaches Dikastes as an IP set - // reference; the rest carry a CIDR. Measured: 4,644 of 18,675. - egressIPSetRuleFraction = 0.25 - - // Where in the walk the rule that matches sits, as a fraction of the whole tier. Real - // flows match at every depth; this picks one representative depth so the two matching - // cases are comparable with each other. - egressTargetDepth = 0.65 - - // The flow. The denied case keeps this destination, which no rule's CIDR contains; the - // matching cases replace it with an address inside the target rule's CIDR. - egressSourceIP = "192.0.2.10" - egressDeniedIP = "198.51.100.20" - egressSourcePort = 45000 ) -// egressPortWeights reproduces the measured head of the port distribution: the fraction of -// rules whose destination ports include each of these. 443 appears in 3,395 of 18,675 rules, -// 11001 in 2,289, 27054 in 2,281 and 80 in 1,505. Rules not drawing a head port get ports from -// a long tail of otherwise-unique values, so a tail port is shared by only a rule or two. -var egressPortWeights = []struct { - port int32 - fraction float64 -}{ - {443, 0.182}, - {11001, 0.123}, - {27054, 0.122}, - {80, 0.081}, -} - -// egressPortsPerRule is the measured spread of destination ports per rule: median 2, mean ~5. -var egressPortsPerRule = []struct { - count int - fraction float64 -}{ - {1, 0.25}, - {2, 0.45}, - {3, 0.15}, - {8, 0.10}, - {20, 0.05}, -} - func BenchmarkEvaluateEgressAllowList(b *testing.B) { // A flow on a port few rules share: rejected on the port comparison nearly everywhere. b.Run("TailPort", func(b *testing.B) { @@ -113,44 +56,34 @@ func BenchmarkEvaluateEgressAllowList(b *testing.B) { // egressCase is what one benchmark case measures: the flow, how many rules the walk is // expected to visit, and whether a rule is expected to match it. type egressCase struct { - flow *MockFlow + flow *policyscale.Flow rulesWalked int matches bool } -// egressCaseFunc builds one case from the fixture's target rule. -type egressCaseFunc func(target egressTarget) egressCase +// egressCaseFunc builds one case from the fixture and its target rule. +type egressCaseFunc func(fx *policyscale.Fixture, target *policyscale.EgressTarget) egressCase -func egressTailPortFlow(target egressTarget) egressCase { +func egressTailPortFlow(_ *policyscale.Fixture, target *policyscale.EgressTarget) egressCase { return egressCase{ - flow: egressFlow(target.addrInCIDR, target.tailPort), - rulesWalked: target.rulesWalked, + flow: policyscale.NewFlow(policyscale.SourceIP, policyscale.DefaultSourcePort, target.AddrInCIDR, int(target.TailPort)), + rulesWalked: target.RulesWalked, matches: true, } } -func egressPopularPortFlow(target egressTarget) egressCase { +func egressPopularPortFlow(_ *policyscale.Fixture, target *policyscale.EgressTarget) egressCase { return egressCase{ - flow: egressFlow(target.addrInCIDR, egressPortWeights[0].port), - rulesWalked: target.rulesWalked, + flow: policyscale.NewFlow(policyscale.SourceIP, policyscale.DefaultSourcePort, target.AddrInCIDR, int(target.PopularPort)), + rulesWalked: target.RulesWalked, matches: true, } } -func egressDeniedFlow(_ egressTarget) egressCase { +func egressDeniedFlow(fx *policyscale.Fixture, _ *policyscale.EgressTarget) egressCase { return egressCase{ - flow: egressFlow(egressDeniedIP, egressPortWeights[0].port), - rulesWalked: egressNumPolicies * egressRulesPerPolicy, - } -} - -func egressFlow(destIP string, destPort int32) *MockFlow { - return &MockFlow{ - SourceIP: net.ParseIP(egressSourceIP), - DestIP: net.ParseIP(destIP), - SourcePort: egressSourcePort, - DestPort: int(destPort), - Protocol: 6, // TCP + flow: fx.DeniedFlow(policyscale.Egress), + rulesWalked: fx.Rules(policyscale.Egress), } } @@ -158,8 +91,9 @@ func benchEvaluateEgressAllowList(b *testing.B, caseFor egressCaseFunc) { _, restoreLogging := withBenchLogging(log.WarnLevel) defer restoreLogging() - store, ep, target := buildEgressAllowListStore() - c := caseFor(target) + fx := policyscale.Build(policyscale.EgressAllowList()) + store, ep, target := fx.NewStore(), fx.Endpoint(), fx.EgressTarget() + c := caseFor(fx, target) // Pre-flight outside the timed loop: prove the walk is the one the case intends, so that // a fixture change cannot silently turn a full walk into an early exit. @@ -168,8 +102,8 @@ func benchEvaluateEgressAllowList(b *testing.B, caseFor egressCaseFunc) { b.Fatalf("evaluation failed: %v", err) } if c.matches { - if len(trace) != 1 || trace[0].Action != rules.RuleActionAllow || trace[0].Index != target.ruleIndex { - b.Fatalf("expected an allow from the target rule at index %d, got %v", target.ruleIndex, trace) + if len(trace) != 1 || trace[0].Action != rules.RuleActionAllow || trace[0].Index != target.RuleIndex { + b.Fatalf("expected an allow from the target rule at index %d, got %v", target.RuleIndex, trace) } } else if len(trace) != 1 || trace[0].Action != rules.RuleActionDeny || trace[0].Index != tierDefaultActionIndex { b.Fatalf("expected a full walk ending in the tier default deny, got %v", trace) @@ -184,125 +118,3 @@ func benchEvaluateEgressAllowList(b *testing.B, caseFor egressCaseFunc) { b.ReportMetric(float64(c.rulesWalked), "rules/op") b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N)/float64(c.rulesWalked), "ns/rule") } - -// egressTarget describes the one rule in the fixture that a matching flow is built to hit. -type egressTarget struct { - ruleIndex int // Index of the rule within its policy, for the trace assertion. - addrInCIDR string // An address inside the rule's destination CIDR. - tailPort int32 // A port on the rule that few other rules share. - rulesWalked int // Rules visited before and including it. -} - -// buildEgressAllowListStore builds the policy store and the endpoint whose single tier applies -// every policy, and returns the target rule a matching flow is aimed at. -// -// Destination CIDRs are handed out sequentially rather than at random so that no rule except -// the target can contain the matching flow's address: a second rule matching earlier would -// shorten the walk and make the case measure something other than what it claims to. -func buildEgressAllowListStore() (*policystore.PolicyStore, *proto.WorkloadEndpoint, egressTarget) { - rng := rand.New(rand.NewSource(20200)) - store := policystore.NewPolicyStore() - setIDs := makeEgressIPSets(store) - - numRules := egressNumPolicies * egressRulesPerPolicy - targetRule := int(float64(numRules) * egressTargetDepth) - - tier := &proto.TierInfo{Name: "perimeter", DefaultAction: "Deny"} - var target egressTarget - nextCIDR := 0 - ruleIdx := 0 - for i := 0; i < egressNumPolicies; i++ { - policyID := &proto.PolicyID{Name: fmt.Sprintf("egress-%03d", i), Kind: v3.KindGlobalNetworkPolicy} - policy := &proto.Policy{Tier: tier.Name} - for j := 0; j < egressRulesPerPolicy; j++ { - // All rules are Pass bar the target: a rule's action is only consulted once it - // matches, so the action mix does not affect the walk. - rule := &proto.Rule{Action: "pass", DstPorts: makeEgressRulePorts(rng)} - if rng.Float64() < egressIPSetRuleFraction { - rule.DstIpSetIds = []string{setIDs[rng.Intn(len(setIDs))]} - } else { - rule.DstNet = []string{fmt.Sprintf("10.%d.%d.0/24", nextCIDR>>8&0xff, nextCIDR&0xff)} - nextCIDR++ - } - if ruleIdx == targetRule { - // Make the target reachable on address and on a port of its own, and give it - // a distinct action so the trace assertion is unambiguous. - rule.Action = "allow" - rule.DstNet = []string{fmt.Sprintf("10.%d.%d.0/24", nextCIDR>>8&0xff, nextCIDR&0xff)} - rule.DstIpSetIds = nil - // Carry both ports the matching cases use, so they differ only in how many - // rules along the way survive the port comparison. - rule.DstPorts = append(rule.DstPorts, - &proto.PortRange{First: egressTailPort, Last: egressTailPort}, - &proto.PortRange{First: egressPortWeights[0].port, Last: egressPortWeights[0].port}, - ) - target = egressTarget{ - ruleIndex: j, - addrInCIDR: fmt.Sprintf("10.%d.%d.7", nextCIDR>>8&0xff, nextCIDR&0xff), - tailPort: egressTailPort, - rulesWalked: ruleIdx + 1, - } - nextCIDR++ - } - policy.OutboundRules = append(policy.OutboundRules, rule) - ruleIdx++ - } - store.PolicyByID[types.ProtoToPolicyID(policyID)] = policy - tier.EgressPolicies = append(tier.EgressPolicies, policyID) - } - - ep := &proto.WorkloadEndpoint{Tiers: []*proto.TierInfo{tier}} - return store, ep, target -} - -// egressTailPort is outside the range makeEgressRulePorts draws from, so only the target rule -// carries it — the extreme of the measured tail, where 90% of ports are on 5 rules or fewer. -const egressTailPort = 60999 - -// makeEgressRulePorts picks a rule's destination ports: the measured head with its measured -// frequency, topped up from a wide tail. -func makeEgressRulePorts(rng *rand.Rand) []*proto.PortRange { - var ports []int32 - for _, w := range egressPortWeights { - if rng.Float64() < w.fraction { - ports = append(ports, w.port) - } - } - for _, n := range egressPortsPerRule { - if rng.Float64() < n.fraction { - for len(ports) < n.count { - ports = append(ports, int32(1024+rng.Intn(55000))) - } - break - } - } - if len(ports) == 0 { - ports = append(ports, int32(1024+rng.Intn(55000))) - } - - ranges := make([]*proto.PortRange, 0, len(ports)) - for _, p := range ports { - ranges = append(ranges, &proto.PortRange{First: p, Last: p}) - } - return ranges -} - -// makeEgressIPSets populates the store with the NET sets the selector-matching rules reference: -// 3,862 sets at a median of 3 members, none containing the flow's addresses. Members are /32s -// from 10.128.0.0/9, kept clear of the rule CIDRs handed out from 10.0.0.0/9. -func makeEgressIPSets(store *policystore.PolicyStore) []string { - const numSets = 3862 - ids := make([]string, 0, numSets) - member := 0 - for i := 0; i < numSets; i++ { - id := fmt.Sprintf("s:egress-%04d", i) - s := policystore.NewIPSet(proto.IPSetUpdate_NET) - for j := 0; j < 3; j++ { - s.AddString(fmt.Sprintf("10.%d.%d.%d/32", 128+(member>>16&0x7f), member>>8&0xff, member&0xff)) - member++ - } - store.IPSetByID[id] = s - ids = append(ids, id) - } - return ids -} diff --git a/app-policy/checker/bench_test.go b/app-policy/checker/bench_test.go index dcb5e510d5d..2b83025808d 100644 --- a/app-policy/checker/bench_test.go +++ b/app-policy/checker/bench_test.go @@ -14,12 +14,12 @@ package checker -// Benchmark for Evaluate() at the scale observed in a large production deployment: -// hundreds of "baseline" policies that apply to every endpoint, and thousands of IP -// sets. All data is synthetic; the scale parameters and the IP set size distribution -// are anonymized measurements taken from the deployment's diagnostics. +// Benchmarks for Evaluate() at the scale observed in a large production deployment. The policy +// sets come from app-policy/policyscale, which also renders them as Calico resources and drives +// the collector-level benchmark, so the numbers here, there and on a node describe the same sets. // -// Run with: +// BenchmarkEvaluateBaselinePolicyScale: hundreds of "baseline" policies that apply to every +// endpoint, and thousands of IP sets. Run with: // // go test ./app-policy/checker/ -run '^$' -bench BenchmarkEvaluateBaselinePolicyScale \ // -benchmem -benchtime 100x -cpu 1 @@ -32,12 +32,12 @@ package checker // measured ns/op to judge whether a node is evaluation-bound or pacing on something // else. Note that the benchmark discards log output, so a real deployment pays the // log write on top of the formatting cost measured here. +// +// BenchmarkEvaluateComposite applies both measured shapes to one endpoint: the reference set the +// PMREQ-954 target (10k flows/s per node, so at most 100 µs per evaluation) is quoted against. import ( - "fmt" "io" - "math/rand" - "net" "strings" "sync/atomic" "testing" @@ -46,6 +46,7 @@ import ( v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" + "github.com/projectcalico/calico/app-policy/policyscale" "github.com/projectcalico/calico/app-policy/policystore" "github.com/projectcalico/calico/felix/calc" "github.com/projectcalico/calico/felix/proto" @@ -54,97 +55,88 @@ import ( "github.com/projectcalico/calico/lib/logrusr" ) -// baselinePolicyScaleParams describes a policy store dominated by "baseline" policies: -// every policy applies to the endpoint, and almost every rule is a Pass. The defaults -// are the anonymized per-node scale measured in a large production deployment. -type baselinePolicyScaleParams struct { - seed int64 - numPolicies int - rulesPerPolicy int - numDenyRules int // Rules with a "deny" action, spread at random; the rest are "pass". - numAllowRules int // As numDenyRules, but "allow". - ipsetRefFraction float64 // Fraction of rules that reference an IP set. - numMissingIPSets int // Referenced IP sets to delete from the store ("IPSet not found" storm). -} - -func defaultBaselinePolicyScaleParams() baselinePolicyScaleParams { - return baselinePolicyScaleParams{ - seed: 20200, - numPolicies: 294, - rulesPerPolicy: 68, - numDenyRules: 10, - numAllowRules: 1, - ipsetRefFraction: 0.242, // ~4,838 IP set references across 294*68 rules. - } -} - -// ipSetSizeHistogram is the per-node IP set size distribution measured in the same -// deployment: 3,708 sets, ~256k members in total, dominated by tiny sets with a long -// tail of large ones. -var ipSetSizeHistogram = []struct{ numSets, minSize, maxSize int }{ - {4, 0, 0}, - {3035, 1, 9}, - {446, 10, 99}, - {64, 100, 999}, - {158, 1000, 9999}, - {1, 54566, 54566}, -} - -// The benchmark flow. TEST-NET addresses; generated IP set members and rule CIDRs all -// come from 10.0.0.0/8, so no rule ever matches on address and the walk covers the -// whole policy set. -const ( - benchSourceIP = "192.0.2.10" - benchDestIP = "198.51.100.20" - benchSourcePort = 45000 - benchDestPort = 8080 - - // benchSentinelIPSetID is a set containing the flow's addresses; see makeScaleRule. - benchSentinelIPSetID = "s:bench-sentinel" -) - // benchTraceSink prevents the compiler from eliminating the Evaluate call. var benchTraceSink []*calc.RuleID func BenchmarkEvaluateBaselinePolicyScale(b *testing.B) { b.Run("AllSetsPresent", func(b *testing.B) { - benchEvaluateBaselinePolicyScale(b, defaultBaselinePolicyScaleParams(), log.WarnLevel, false) + benchEvaluateBaselinePolicyScale(b, policyscale.Baseline(), log.WarnLevel, false) }) b.Run("MissingSets", func(b *testing.B) { - p := defaultBaselinePolicyScaleParams() - p.numMissingIPSets = 8 - benchEvaluateBaselinePolicyScale(b, p, log.WarnLevel, false) + spec := policyscale.Baseline() + spec.Baseline.MissingIPSets = 8 + benchEvaluateBaselinePolicyScale(b, spec, log.WarnLevel, false) }) // As MissingSets but with warnings disabled, to isolate the cost of formatting the // "IPSet not found" warnings. b.Run("MissingSetsLogsOff", func(b *testing.B) { - p := defaultBaselinePolicyScaleParams() - p.numMissingIPSets = 8 - benchEvaluateBaselinePolicyScale(b, p, log.ErrorLevel, false) + spec := policyscale.Baseline() + spec.Baseline.MissingIPSets = 8 + benchEvaluateBaselinePolicyScale(b, spec, log.ErrorLevel, false) }) // Fixed per-Evaluate overhead: the first rule of the first policy matches, so the // walk short-circuits immediately. b.Run("MatchEarly", func(b *testing.B) { - benchEvaluateBaselinePolicyScale(b, defaultBaselinePolicyScaleParams(), log.WarnLevel, true) + benchEvaluateBaselinePolicyScale(b, policyscale.Baseline(), log.WarnLevel, true) }) } -func benchEvaluateBaselinePolicyScale(b *testing.B, p baselinePolicyScaleParams, level log.Level, matchEarly bool) { +// BenchmarkEvaluateComposite measures the reference set in the three cases that matter for the +// collector: an ingress flow that misses every baseline rule, an egress flow that misses every +// allow-list rule, and an egress flow that matches the target rule on a port few rules share. +func BenchmarkEvaluateComposite(b *testing.B) { + _, restoreLogging := withBenchLogging(log.WarnLevel) + defer restoreLogging() + + fx := policyscale.Build(policyscale.Composite()) + store, ep := fx.NewStore(), fx.Endpoint() + target := fx.EgressTarget() + + cases := []struct { + name string + dir rules.RuleDir + flow *policyscale.Flow + walk int + final rules.RuleAction + index int + }{ + {"IngressMissAll", rules.RuleDirIngress, fx.DeniedFlow(policyscale.Ingress), fx.Rules(policyscale.Ingress), rules.RuleActionDeny, tierDefaultActionIndex}, + {"EgressMissAll", rules.RuleDirEgress, fx.DeniedFlow(policyscale.Egress), fx.Rules(policyscale.Egress), rules.RuleActionDeny, tierDefaultActionIndex}, + {"EgressTailPort", rules.RuleDirEgress, policyscale.NewFlow(policyscale.SourceIP, policyscale.DefaultSourcePort, target.AddrInCIDR, int(target.TailPort)), target.RulesWalked, rules.RuleActionAllow, target.RuleIndex}, + } + for _, c := range cases { + b.Run(c.name, func(b *testing.B) { + trace, err := Evaluate(StagedAsEnforced, c.dir, store, ep, c.flow) + if err != nil { + b.Fatalf("evaluation failed: %v", err) + } + if len(trace) != 1 || trace[0].Action != c.final || trace[0].Index != c.index { + b.Fatalf("expected %v at index %d, got %v", c.final, c.index, trace) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + benchTraceSink, _ = Evaluate(StagedAsEnforced, c.dir, store, ep, c.flow) + } + b.StopTimer() + b.ReportMetric(float64(c.walk), "rules/op") + b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N)/float64(c.walk), "ns/rule") + }) + } +} + +func benchEvaluateBaselinePolicyScale(b *testing.B, spec policyscale.Spec, level log.Level, matchEarly bool) { logger := log.StandardLogger() counter, restoreLogging := withBenchLogging(level) defer restoreLogging() - store, ep, expectedWarns := buildBaselinePolicyStore(p) + fx := policyscale.Build(spec) + store, ep := fx.NewStore(), fx.Endpoint() + expectedWarns := fx.MissingSetReferences() if matchEarly { addMatchEarlyPolicy(store, ep) } - flow := &MockFlow{ - SourceIP: net.ParseIP(benchSourceIP), - DestIP: net.ParseIP(benchDestIP), - SourcePort: benchSourcePort, - DestPort: benchDestPort, - Protocol: 6, // TCP - } + flow := fx.DeniedFlow(policyscale.Ingress) // Pre-flight outside the timed loop: prove the walk is the intended one and that // the warning count matches the analytic count, so that warnings/op is exact. @@ -171,130 +163,13 @@ func benchEvaluateBaselinePolicyScale(b *testing.B, p baselinePolicyScaleParams, } b.StopTimer() b.ReportMetric(float64(counter.count.Load())/float64(b.N), "warnings/op") - rulesWalked := p.numPolicies * p.rulesPerPolicy + rulesWalked := fx.Rules(policyscale.Ingress) if matchEarly { rulesWalked = 1 } b.ReportMetric(float64(rulesWalked), "rules/op") } -// buildBaselinePolicyStore builds a policy store at the given scale, plus an endpoint -// whose single "perimeter" tier applies every policy. It returns the number of rule -// references to deleted (missing) IP sets, which is exactly the number of "IPSet not -// found" warnings one Evaluate of a non-matching flow emits. -func buildBaselinePolicyStore(p baselinePolicyScaleParams) (*policystore.PolicyStore, *proto.WorkloadEndpoint, int) { - rng := rand.New(rand.NewSource(p.seed)) - store := policystore.NewPolicyStore() - setIDs := makeScaleIPSets(rng, store) - - // Pick the rule slots that get the few non-Pass actions. - numRules := p.numPolicies * p.rulesPerPolicy - specialAction := map[int]string{} - for len(specialAction) < p.numDenyRules { - specialAction[rng.Intn(numRules)] = "deny" - } - for n := 0; n < p.numAllowRules; { - slot := rng.Intn(numRules) - if _, ok := specialAction[slot]; !ok { - specialAction[slot] = "allow" - n++ - } - } - - tier := &proto.TierInfo{Name: "perimeter", DefaultAction: "Deny"} - var referencedIDs []string - refCount := map[string]int{} - ruleIdx := 0 - for i := 0; i < p.numPolicies; i++ { - policyID := &proto.PolicyID{Name: fmt.Sprintf("policy-%03d", i), Kind: v3.KindGlobalNetworkPolicy} - policy := &proto.Policy{Tier: tier.Name} - for j := 0; j < p.rulesPerPolicy; j++ { - action := specialAction[ruleIdx] - if action == "" { - action = "pass" - } - rule, refID := makeScaleRule(rng, setIDs, action, p.ipsetRefFraction) - policy.InboundRules = append(policy.InboundRules, rule) - if refID != "" { - if refCount[refID] == 0 { - referencedIDs = append(referencedIDs, refID) - } - refCount[refID]++ - } - ruleIdx++ - } - store.PolicyByID[types.ProtoToPolicyID(policyID)] = policy - tier.IngressPolicies = append(tier.IngressPolicies, policyID) - } - - // Delete some referenced sets to reproduce the "IPSet not found" storm. - rng.Shuffle(len(referencedIDs), func(a, b int) { - referencedIDs[a], referencedIDs[b] = referencedIDs[b], referencedIDs[a] - }) - expectedWarns := 0 - for _, id := range referencedIDs[:p.numMissingIPSets] { - delete(store.IPSetByID, id) - expectedWarns += refCount[id] - } - - ep := &proto.WorkloadEndpoint{Tiers: []*proto.TierInfo{tier}} - return store, ep, expectedWarns -} - -// makeScaleIPSets populates the store with NET-type IP sets (the dominant type in the -// measured deployment: selector and networkset derived sets) following the measured -// size histogram. Members are unique /32s from 10.0.0.0/8. -func makeScaleIPSets(rng *rand.Rand, store *policystore.PolicyStore) []string { - sentinel := policystore.NewIPSet(proto.IPSetUpdate_NET) - sentinel.AddString(benchSourceIP + "/32") - sentinel.AddString(benchDestIP + "/32") - store.IPSetByID[benchSentinelIPSetID] = sentinel - - var ids []string - member := 0 - for _, bucket := range ipSetSizeHistogram { - for i := 0; i < bucket.numSets; i++ { - id := fmt.Sprintf("s:bench-%04d", len(ids)) - s := policystore.NewIPSet(proto.IPSetUpdate_NET) - size := bucket.minSize + rng.Intn(bucket.maxSize-bucket.minSize+1) - for j := 0; j < size; j++ { - s.AddString(fmt.Sprintf("10.%d.%d.%d/32", member>>16&0xff, member>>8&0xff, member&0xff)) - member++ - } - store.IPSetByID[id] = s - ids = append(ids, id) - } - } - return ids -} - -// makeScaleRule builds a rule that never matches the benchmark flow. It returns the -// referenced IP set ID, or "" for a rule with no reference. -// -// A rule that references an IP set must reach the set lookup whatever order match() -// evaluates criteria in, and must still miss when the referenced set is absent from -// the store (an absent set is skipped rather than treated as a non-match). Both hold -// by pairing the reference with a negated reference to benchSentinelIPSetID, which -// contains the flow's addresses: if the referenced set is present the lookup misses -// and evaluation stops there; if it is absent the sentinel makes the rule miss. -func makeScaleRule(rng *rand.Rand, setIDs []string, action string, refFraction float64) (*proto.Rule, string) { - rule := &proto.Rule{Action: action} - if rng.Float64() < refFraction { - id := setIDs[rng.Intn(len(setIDs))] - if rng.Intn(2) == 0 { - rule.SrcIpSetIds = []string{id} - rule.NotSrcIpSetIds = []string{benchSentinelIPSetID} - } else { - rule.DstIpSetIds = []string{id} - rule.NotDstIpSetIds = []string{benchSentinelIPSetID} - } - return rule, id - } - // No IP set reference: guard with a non-matching destination port. - rule.DstPorts = []*proto.PortRange{{First: 65001, Last: 65001}} - return rule, "" -} - // addMatchEarlyPolicy prepends a policy whose first rule matches any flow. func addMatchEarlyPolicy(store *policystore.PolicyStore, ep *proto.WorkloadEndpoint) { policyID := &proto.PolicyID{Name: "policy-match-early", Kind: v3.KindGlobalNetworkPolicy} diff --git a/app-policy/policyscale/build.go b/app-policy/policyscale/build.go new file mode 100644 index 00000000000..c1d2f09eed1 --- /dev/null +++ b/app-policy/policyscale/build.go @@ -0,0 +1,442 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policyscale + +import ( + "fmt" + "math/rand" + "net/netip" + + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + + "github.com/projectcalico/calico/felix/proto" +) + +const ( + // SentinelIPSetID names the set that holds the denied flow's addresses. Baseline rules that + // reference an IP set also carry a negated reference to it; see BaselineParams. + SentinelIPSetID = "policyscale-sentinel" + + baselineIPSetPrefix = "policyscale-base-" + egressIPSetPrefix = "policyscale-egress-" + + // BaselineGuardPort is the destination port that guards baseline rules with no IP set + // reference. No generated flow uses it, so those rules never match. + BaselineGuardPort int32 = 65001 + + // EgressTailPort is outside the range egress rules draw their ports from, so only the target + // rule carries it: the extreme of the measured tail, where 90% of ports are on 5 rules or + // fewer. + EgressTailPort int32 = 60999 + + // egressTailPortMin/Range bound the long tail of otherwise-unique destination ports. + egressTailPortMin = 1024 + egressTailPortRange = 55000 +) + +// Fixture is a built policy set: the model the oracle and flow generators reason about, plus the +// proto policies the store and endpoint are made from. +type Fixture struct { + Spec Spec + + tiers []*tierModel + sets map[string]*ipSetModel + setOrder []string + + missingRefs int + egressTarget *EgressTarget +} + +// EgressTarget describes the one egress rule a matching flow is built to hit when +// EgressParams.TargetDepth is set. +type EgressTarget struct { + Policy string // Policy name, as the trace reports it. + RuleIndex int // Index of the rule within its policy. + Ordinal int // Position of the rule in the egress walk, from 0. + RulesWalked int // Rules visited before and including it. + AddrInCIDR string // An address inside the rule's destination CIDR. + TailPort int32 // A port only the target carries. + PopularPort int32 // The most popular port, which the target also carries. +} + +type tierModel struct { + name string + defaultAction string // "Deny" or "Pass", as TierInfo.DefaultAction spells it. + policies [2][]*policyModel +} + +type policyModel struct { + name string + rules []*ruleModel + + id *proto.PolicyID + policy *proto.Policy +} + +// ruleModel is what the generator knows about a rule. Every criterion is ANDed; an unset one is +// unconstrained. The oracle evaluates this, never the proto. +type ruleModel struct { + action string // "allow", "deny" or "pass". + dstPorts []int32 + dstNet netip.Prefix + srcSet, notSrcSet, dstSet, notDstSet string +} + +type ipSetModel struct { + id string + members []string // CIDR strings, as the store and the resources take them. + addrs map[netip.Addr]struct{} + missing bool +} + +// Build materialises a Spec. Each preset draws from its own generator seeded with Spec.Seed, so a +// preset builds the same rules whether it is used alone or in a composite. +func Build(spec Spec) *Fixture { + fx := &Fixture{Spec: spec, sets: map[string]*ipSetModel{}} + if spec.Baseline != nil { + fx.buildBaseline(rand.New(rand.NewSource(spec.Seed)), *spec.Baseline) + } + if spec.Egress != nil { + fx.buildEgress(rand.New(rand.NewSource(spec.Seed)), *spec.Egress) + } + for _, t := range fx.tiers { + for dir, policies := range t.policies { + for _, p := range policies { + p.buildProto(t.name, Direction(dir)) + } + } + } + return fx +} + +func (fx *Fixture) buildBaseline(rng *rand.Rand, p BaselineParams) { + setIDs := fx.makeBaselineIPSets(rng, p.SizeHistogram) + + // Pick the rule slots that get the few non-Pass actions. + numRules := p.Policies * p.RulesPerPolicy + specialAction := map[int]string{} + for len(specialAction) < p.DenyRules { + specialAction[rng.Intn(numRules)] = "deny" + } + for n := 0; n < p.AllowRules; { + slot := rng.Intn(numRules) + if _, ok := specialAction[slot]; !ok { + specialAction[slot] = "allow" + n++ + } + } + + tier := fx.tier(p.Tier, "Deny") + var referencedIDs []string + refCount := map[string]int{} + ruleIdx := 0 + for i := 0; i < p.Policies; i++ { + pol := &policyModel{name: fmt.Sprintf("policy-%03d", i)} + for j := 0; j < p.RulesPerPolicy; j++ { + action := specialAction[ruleIdx] + if action == "" { + action = "pass" + } + r := &ruleModel{action: action} + var refID string + if rng.Float64() < p.IPSetRefFraction { + refID = setIDs[rng.Intn(len(setIDs))] + if rng.Intn(2) == 0 { + r.srcSet, r.notSrcSet = refID, SentinelIPSetID + } else { + r.dstSet, r.notDstSet = refID, SentinelIPSetID + } + } else { + r.dstPorts = []int32{BaselineGuardPort} + } + pol.rules = append(pol.rules, r) + if refID != "" { + if refCount[refID] == 0 { + referencedIDs = append(referencedIDs, refID) + } + refCount[refID]++ + } + ruleIdx++ + } + tier.policies[Ingress] = append(tier.policies[Ingress], pol) + } + + // Leave some referenced sets out of the store to reproduce the "IPSet not found" storm. + rng.Shuffle(len(referencedIDs), func(a, b int) { + referencedIDs[a], referencedIDs[b] = referencedIDs[b], referencedIDs[a] + }) + for _, id := range referencedIDs[:min(p.MissingIPSets, len(referencedIDs))] { + fx.sets[id].missing = true + fx.missingRefs += refCount[id] + } +} + +// makeBaselineIPSets adds the sentinel set and the NET sets the baseline rules reference, +// following the size histogram. Members are unique /32s from 10.0.0.0/9. +func (fx *Fixture) makeBaselineIPSets(rng *rand.Rand, hist []SizeBucket) []string { + fx.addSet(SentinelIPSetID, []string{SourceIP + "/32", DeniedDestIP + "/32"}) + + var ids []string + member := 0 + for _, bucket := range hist { + for i := 0; i < bucket.Sets; i++ { + id := fmt.Sprintf("%s%04d", baselineIPSetPrefix, len(ids)) + size := bucket.MinSize + rng.Intn(bucket.MaxSize-bucket.MinSize+1) + members := make([]string, 0, size) + for j := 0; j < size; j++ { + members = append(members, fmt.Sprintf("10.%d.%d.%d/32", member>>16&0x7f, member>>8&0xff, member&0xff)) + member++ + } + fx.addSet(id, members) + ids = append(ids, id) + } + } + return ids +} + +func (fx *Fixture) buildEgress(rng *rand.Rand, p EgressParams) { + setIDs := fx.makeEgressIPSets(p.IPSets, p.MembersPerSet) + + numRules := p.Policies * p.RulesPerPolicy + targetRule := -1 + if p.TargetDepth > 0 { + targetRule = int(float64(numRules) * p.TargetDepth) + } + + tier := fx.tier(p.Tier, "Deny") + nextCIDR := 0 + ruleIdx := 0 + for i := 0; i < p.Policies; i++ { + pol := &policyModel{name: fmt.Sprintf("egress-%03d", i)} + for j := 0; j < p.RulesPerPolicy; j++ { + // All rules are Pass bar the target: a rule's action is only consulted once it + // matches, so the action mix does not affect the walk. + r := &ruleModel{action: "pass", dstPorts: makeEgressRulePorts(rng, p)} + if rng.Float64() < p.IPSetRuleFraction { + r.dstSet = setIDs[rng.Intn(len(setIDs))] + } else { + r.dstNet = egressCIDR(nextCIDR) + nextCIDR++ + } + if ruleIdx == targetRule { + // Make the target reachable on address and on a port of its own, and give it a + // distinct action so a trace assertion is unambiguous. It carries both the tail + // port and the popular port, so flows aimed at it differ only in how many rules + // along the way survive the port comparison. + r.action = "allow" + r.dstNet = egressCIDR(nextCIDR) + r.dstSet = "" + r.dstPorts = append(r.dstPorts, EgressTailPort, p.PortWeights[0].Port) + fx.egressTarget = &EgressTarget{ + Policy: pol.name, + RuleIndex: j, + Ordinal: ruleIdx, + RulesWalked: ruleIdx + 1, + AddrInCIDR: egressAddrInCIDR(nextCIDR), + TailPort: EgressTailPort, + PopularPort: p.PortWeights[0].Port, + } + nextCIDR++ + } + pol.rules = append(pol.rules, r) + ruleIdx++ + } + tier.policies[Egress] = append(tier.policies[Egress], pol) + } +} + +// makeEgressIPSets adds the NET sets the selector-matching egress rules reference. Members are +// /32s from 10.128.0.0/9, kept clear of the rule CIDRs handed out from 10.0.0.0/9. +func (fx *Fixture) makeEgressIPSets(numSets, membersPerSet int) []string { + ids := make([]string, 0, numSets) + member := 0 + for i := 0; i < numSets; i++ { + id := fmt.Sprintf("%s%04d", egressIPSetPrefix, i) + members := make([]string, 0, membersPerSet) + for j := 0; j < membersPerSet; j++ { + members = append(members, fmt.Sprintf("10.%d.%d.%d/32", 128+(member>>16&0x7f), member>>8&0xff, member&0xff)) + member++ + } + fx.addSet(id, members) + ids = append(ids, id) + } + return ids +} + +// makeEgressRulePorts picks a rule's destination ports: the measured head with its measured +// frequency, topped up from a wide tail. +func makeEgressRulePorts(rng *rand.Rand, p EgressParams) []int32 { + var ports []int32 + for _, w := range p.PortWeights { + if rng.Float64() < w.Fraction { + ports = append(ports, w.Port) + } + } + for _, n := range p.PortsPerRule { + if rng.Float64() < n.Fraction { + for len(ports) < n.Count { + ports = append(ports, int32(egressTailPortMin+rng.Intn(egressTailPortRange))) + } + break + } + } + if len(ports) == 0 { + ports = append(ports, int32(egressTailPortMin+rng.Intn(egressTailPortRange))) + } + return ports +} + +func egressCIDR(n int) netip.Prefix { + return netip.MustParsePrefix(fmt.Sprintf("10.%d.%d.0/24", n>>8&0x7f, n&0xff)) +} + +func egressAddrInCIDR(n int) string { + return fmt.Sprintf("10.%d.%d.7", n>>8&0x7f, n&0xff) +} + +func (fx *Fixture) tier(name, defaultAction string) *tierModel { + for _, t := range fx.tiers { + if t.name == name { + return t + } + } + t := &tierModel{name: name, defaultAction: defaultAction} + fx.tiers = append(fx.tiers, t) + return t +} + +func (fx *Fixture) addSet(id string, members []string) { + if _, ok := fx.sets[id]; ok { + return + } + s := &ipSetModel{id: id, members: members, addrs: make(map[netip.Addr]struct{}, len(members))} + for _, m := range members { + s.addrs[netip.MustParsePrefix(m).Addr().Unmap()] = struct{}{} + } + fx.sets[id] = s + fx.setOrder = append(fx.setOrder, id) +} + +func (p *policyModel) buildProto(tier string, dir Direction) { + protoRules := make([]*proto.Rule, len(p.rules)) + for i, r := range p.rules { + protoRules[i] = r.proto() + } + p.id = &proto.PolicyID{Name: p.name, Kind: v3.KindGlobalNetworkPolicy} + p.policy = &proto.Policy{Tier: tier} + if dir == Egress { + p.policy.OutboundRules = protoRules + } else { + p.policy.InboundRules = protoRules + } +} + +func (r *ruleModel) proto() *proto.Rule { + pr := &proto.Rule{Action: r.action} + for _, port := range r.dstPorts { + pr.DstPorts = append(pr.DstPorts, &proto.PortRange{First: port, Last: port}) + } + if r.dstNet.IsValid() { + pr.DstNet = []string{r.dstNet.String()} + } + if r.srcSet != "" { + pr.SrcIpSetIds = []string{r.srcSet} + } + if r.notSrcSet != "" { + pr.NotSrcIpSetIds = []string{r.notSrcSet} + } + if r.dstSet != "" { + pr.DstIpSetIds = []string{r.dstSet} + } + if r.notDstSet != "" { + pr.NotDstIpSetIds = []string{r.notDstSet} + } + return pr +} + +// Tiers returns the tier names in evaluation order. +func (fx *Fixture) Tiers() []string { + names := make([]string, len(fx.tiers)) + for i, t := range fx.tiers { + names[i] = t.name + } + return names +} + +// Policies returns the number of policies with rules in the given direction. +func (fx *Fixture) Policies(dir Direction) int { + n := 0 + for _, t := range fx.tiers { + n += len(t.policies[dir]) + } + return n +} + +// Rules returns the number of rules a flow in the given direction walks when nothing matches. +func (fx *Fixture) Rules(dir Direction) int { + n := 0 + for _, t := range fx.tiers { + for _, p := range t.policies[dir] { + n += len(p.rules) + } + } + return n +} + +// IPSets returns the number of IP sets the policies reference, including any left out of the +// store; MissingIPSets says how many of those are left out. +func (fx *Fixture) IPSets() int { return len(fx.setOrder) } + +func (fx *Fixture) MissingIPSets() int { + n := 0 + for _, id := range fx.setOrder { + if fx.sets[id].missing { + n++ + } + } + return n +} + +// IPSetMembers returns the total number of members across the IP sets present in the store. +func (fx *Fixture) IPSetMembers() int { + n := 0 + for _, id := range fx.setOrder { + if s := fx.sets[id]; !s.missing { + n += len(s.members) + } + } + return n +} + +// MissingSetReferences returns the number of rule references to sets left out of the store, +// which is exactly the number of "IPSet not found" warnings one evaluation of a flow that walks +// the whole baseline set emits. +func (fx *Fixture) MissingSetReferences() int { return fx.missingRefs } + +// EgressTarget returns the target rule, or nil when the egress preset has none. +func (fx *Fixture) EgressTarget() *EgressTarget { return fx.egressTarget } + +// ruleAt returns the policy and rule at the given position of the walk in a direction. +func (fx *Fixture) ruleAt(dir Direction, ordinal int) (*policyModel, *ruleModel, bool) { + for _, t := range fx.tiers { + for _, p := range t.policies[dir] { + if ordinal < len(p.rules) { + return p, p.rules[ordinal], true + } + ordinal -= len(p.rules) + } + } + return nil, nil, false +} diff --git a/app-policy/policyscale/doc.go b/app-policy/policyscale/doc.go new file mode 100644 index 00000000000..87179728b66 --- /dev/null +++ b/app-policy/policyscale/doc.go @@ -0,0 +1,42 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package policyscale generates the large synthetic policy sets used to benchmark and test the +// user-mode policy engine in app-policy/checker and its caller in Felix's flow-log collector. +// +// One generator, two outputs. A Fixture built from a Spec can be materialised as a +// policystore.PolicyStore plus a proto.WorkloadEndpoint (what the engine and the collector +// evaluate against, in unit tests and Go benchmarks) or as Calico resources (Tier, +// GlobalNetworkPolicy, GlobalNetworkSet) that apply the same policy set to a real cluster. Both +// come from the same model, so a number measured at the engine level, at the collector level and +// on a node all describe the same policy set. +// +// The presets are the two rule-set shapes measured in production diagnostics from a large +// deployment, anonymised: +// +// - Baseline: hundreds of "baseline" policies that apply to every endpoint, almost every rule a +// Pass, about a quarter referencing one of thousands of IP sets whose sizes follow the +// measured histogram. See DefaultBaseline. +// - EgressAllowList: a single-tier destination allow-list of ~300 policies and ~18.6k egress +// rules, each matching a destination address (a CIDR for ~75%, a selector-derived IP set for +// ~25%) and a handful of destination ports drawn from the measured port distribution. See +// DefaultEgress. +// - Composite: both applied to one endpoint, the reference set the PMREQ-954 targets are quoted +// against. +// +// The fixture also knows the answer. Fixture.Expect computes the verdict the engine must reach +// for a flow from the generator's own model of each rule, independently of the engine's matching +// code, so it serves as the oracle for differential tests; MatchingFlow, DeniedFlow and Sampler +// produce flows that exercise the walk at chosen depths or following a configurable flow model. +package policyscale diff --git a/app-policy/policyscale/flow.go b/app-policy/policyscale/flow.go new file mode 100644 index 00000000000..369cf09b888 --- /dev/null +++ b/app-policy/policyscale/flow.go @@ -0,0 +1,182 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policyscale + +import ( + "fmt" + "math/rand" + "net" +) + +// The addresses and ports the denied flows use. TEST-NET addresses: generated IP set members and +// rule CIDRs all come from 10.0.0.0/8, so no rule matches them on address and the walk covers the +// whole policy set. +const ( + SourceIP = "192.0.2.10" + DeniedDestIP = "198.51.100.20" + // UnlistedIP is in no generated set and no rule CIDR, and unlike the two above it is not in + // the sentinel set either. + UnlistedIP = "203.0.113.1" + + DefaultSourcePort = 45000 + // BaselineDestPort is the denied ingress flow's destination port; no baseline rule uses it. + BaselineDestPort = 8080 + + ProtocolTCP = 6 +) + +// Flow is an L4 flow with no L7 attributes, as the collector presents flows to the engine. It +// satisfies checker.Flow. +type Flow struct { + SrcIP, DstIP net.IP + SrcPort, DstPort int + Protocol int +} + +func (f *Flow) GetSourceIP() net.IP { return f.SrcIP } +func (f *Flow) GetDestIP() net.IP { return f.DstIP } +func (f *Flow) GetSourcePort() int { return f.SrcPort } +func (f *Flow) GetDestPort() int { return f.DstPort } +func (f *Flow) GetProtocol() int { return f.Protocol } +func (f *Flow) GetHttpMethod() *string { return nil } +func (f *Flow) GetHttpPath() *string { return nil } +func (f *Flow) GetSourcePrincipal() *string { return nil } +func (f *Flow) GetDestPrincipal() *string { return nil } +func (f *Flow) GetSourceLabels() map[string]string { return nil } +func (f *Flow) GetDestLabels() map[string]string { return nil } + +func (f *Flow) String() string { + return fmt.Sprintf("%d %s:%d->%s:%d", f.Protocol, f.SrcIP, f.SrcPort, f.DstIP, f.DstPort) +} + +// NewFlow builds a TCP flow. +func NewFlow(srcIP string, srcPort int, dstIP string, dstPort int) *Flow { + return &Flow{ + SrcIP: net.ParseIP(srcIP), + DstIP: net.ParseIP(dstIP), + SrcPort: srcPort, + DstPort: dstPort, + Protocol: ProtocolTCP, + } +} + +// DeniedFlow returns a flow that no rule in the given direction matches, so an evaluation walks +// the whole policy set and ends in the tier default deny. In the egress direction it uses the most +// popular port, so that the address criteria of every rule sharing that port are evaluated too. +func (fx *Fixture) DeniedFlow(dir Direction) *Flow { + port := BaselineDestPort + if dir == Egress && fx.Spec.Egress != nil && len(fx.Spec.Egress.PortWeights) > 0 { + port = int(fx.Spec.Egress.PortWeights[0].Port) + } + return NewFlow(SourceIP, DefaultSourcePort, DeniedDestIP, port) +} + +// MatchingFlow returns a flow aimed at the rule at the given position of the walk in a direction: +// one that satisfies that rule's criteria. An earlier rule with overlapping criteria may match it +// first; Expect gives the actual verdict. A flow aimed at a rule that cannot match (an empty IP +// set, or a rule whose position is out of range) is a denied flow. +func (fx *Fixture) MatchingFlow(dir Direction, ordinal int) *Flow { + f := fx.DeniedFlow(dir) + _, r, ok := fx.ruleAt(dir, ordinal) + if !ok { + return f + } + if len(r.dstPorts) > 0 { + f.DstPort = int(r.dstPorts[0]) + } + switch { + case r.dstNet.IsValid(): + f.DstIP = net.ParseIP(r.dstNet.Addr().Next().Next().String()) + case r.dstSet != "": + f.DstIP = fx.addrMatching(r.dstSet) + case r.srcSet != "": + f.SrcIP = fx.addrMatching(r.srcSet) + } + return f +} + +// addrMatching returns an address that satisfies a positive reference to the set: its first +// member, or, for a set the store does not hold (the engine skips those), an address outside the +// sentinel set. Nil when the set is present but empty, which no address satisfies. +func (fx *Fixture) addrMatching(setID string) net.IP { + s := fx.sets[setID] + if s.missing { + return net.ParseIP(UnlistedIP) + } + if len(s.members) == 0 { + return nil + } + ip, _, err := net.ParseCIDR(s.members[0]) + if err != nil { + panic(err) + } + return ip +} + +// FlowModel describes the mix of flows a Sampler produces. +type FlowModel struct { + Direction Direction + // MissFraction is the fraction of new flows that match no rule and walk the whole set. + MissFraction float64 + // RepeatFraction is the fraction of flows that repeat an earlier flow's addresses and + // destination port with a new source port, the way clients reconnect. It decides whether a + // verdict cache keyed without the source port can help. + RepeatFraction float64 + // RecentWindow is how many distinct flows repeats are drawn from. Default 1024. + RecentWindow int +} + +// Sampler produces flows following a FlowModel: matching flows are aimed uniformly at every depth +// of the walk, so the verdict mix follows the rule mix. +type Sampler struct { + fx *Fixture + rng *rand.Rand + model FlowModel + recent []*Flow +} + +// NewSampler returns a deterministic sampler for the fixture. +func (fx *Fixture) NewSampler(seed int64, model FlowModel) *Sampler { + if model.RecentWindow <= 0 { + model.RecentWindow = 1024 + } + return &Sampler{fx: fx, rng: rand.New(rand.NewSource(seed)), model: model} +} + +// Next returns the next flow. The returned flow is the caller's to keep. +func (s *Sampler) Next() *Flow { + if len(s.recent) > 0 && s.rng.Float64() < s.model.RepeatFraction { + f := *s.recent[s.rng.Intn(len(s.recent))] + f.SrcPort = s.ephemeralPort() + return &f + } + var f *Flow + if rules := s.fx.Rules(s.model.Direction); rules == 0 || s.rng.Float64() < s.model.MissFraction { + f = s.fx.DeniedFlow(s.model.Direction) + } else { + f = s.fx.MatchingFlow(s.model.Direction, s.rng.Intn(rules)) + } + f.SrcPort = s.ephemeralPort() + if len(s.recent) < s.model.RecentWindow { + s.recent = append(s.recent, f) + } else { + s.recent[s.rng.Intn(len(s.recent))] = f + } + return f +} + +func (s *Sampler) ephemeralPort() int { + return 32768 + s.rng.Intn(28232) +} diff --git a/app-policy/policyscale/oracle.go b/app-policy/policyscale/oracle.go new file mode 100644 index 00000000000..eed096deca9 --- /dev/null +++ b/app-policy/policyscale/oracle.go @@ -0,0 +1,135 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policyscale + +import ( + "fmt" + "net" + "net/netip" + "slices" + "strings" + + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" +) + +// Verdict is one entry of the rule trace the engine is expected to report: a Pass on the way, or +// the final Allow or Deny. It carries the fields of calc.RuleID that identify a rule. +type Verdict struct { + Kind string // GlobalNetworkPolicy, or Profile for the end-of-profiles deny. + Tier string + Policy string + Index int // Rule index in the policy; TierDefaultIndex for a tier or profile default. + Action string // allow, deny or pass. +} + +const ( + // TierDefaultIndex is the index the engine reports for a tier's default action, and for the + // deny that ends an evaluation with no matching profile. + TierDefaultIndex = -1 + // ProfileName is the tier and policy name the engine reports for that deny. + ProfileName = "__PROFILE__" +) + +func (v Verdict) String() string { + return fmt.Sprintf("%s/%s/%s[%d]=%s", v.Kind, v.Tier, v.Policy, v.Index, v.Action) +} + +// Expect returns the trace the engine must report for the flow in the given direction, computed +// from the generator's own model of each rule rather than from the proto the engine reads, so that +// the two are independent. It reproduces the engine's tier walk: policies of a tier in order, +// first matching rule decides; Allow or Deny ends the evaluation; Pass moves to the next tier; a +// tier none of whose policies matched applies its default action; and with no profiles on the +// endpoint, an evaluation that gets past every tier is denied. +func (fx *Fixture) Expect(dir Direction, f *Flow) []Verdict { + var trace []Verdict + for _, t := range fx.tiers { + policies := t.policies[dir] + if len(policies) == 0 { + continue + } + matched := false + Tier: + for _, p := range policies { + for i, r := range p.rules { + if !fx.ruleMatches(r, f) { + continue + } + matched = true + trace = append(trace, Verdict{Kind: v3.KindGlobalNetworkPolicy, Tier: t.name, Policy: p.name, Index: i, Action: r.action}) + if r.action == "pass" { + break Tier + } + return trace + } + } + if !matched { + action := strings.ToLower(t.defaultAction) + trace = append(trace, Verdict{Kind: v3.KindGlobalNetworkPolicy, Tier: t.name, Policy: policies[0].name, Index: TierDefaultIndex, Action: action}) + if action != "pass" { + return trace + } + } + } + return append(trace, Verdict{Kind: v3.KindProfile, Tier: ProfileName, Policy: ProfileName, Index: TierDefaultIndex, Action: "deny"}) +} + +// ruleMatches evaluates a rule's criteria against the flow. The criteria are the ones the +// generator emits; each mirrors the engine's semantics for that field: +// - a protocol outside 1..255 matches no rule; +// - a port list matches when it lists the flow's destination port; +// - a CIDR matches when it contains the destination address; +// - a positive IP set reference requires the address to be a member; a negated one requires it +// not to be; a reference to a set the store does not hold is skipped, as the engine skips it; +// - an address that is not an IP (nil) is a member of no set and inside no CIDR. +func (fx *Fixture) ruleMatches(r *ruleModel, f *Flow) bool { + if f.Protocol < 1 || f.Protocol > 255 { + return false + } + if len(r.dstPorts) > 0 && !slices.Contains(r.dstPorts, int32(f.DstPort)) { + return false + } + if r.dstNet.IsValid() { + a, ok := toAddr(f.DstIP) + if !ok || !r.dstNet.Contains(a) { + return false + } + } + return fx.setAllows(r.srcSet, f.SrcIP, true) && + fx.setAllows(r.notSrcSet, f.SrcIP, false) && + fx.setAllows(r.dstSet, f.DstIP, true) && + fx.setAllows(r.notDstSet, f.DstIP, false) +} + +// setAllows reports whether a (possibly negated) reference to a set lets the address through. +func (fx *Fixture) setAllows(setID string, ip net.IP, positive bool) bool { + if setID == "" { + return true + } + s := fx.sets[setID] + if s.missing { + return true + } + a, ok := toAddr(ip) + if !ok { + return !positive + } + _, member := s.addrs[a] + return member == positive +} + +func toAddr(ip net.IP) (netip.Addr, bool) { + a, ok := netip.AddrFromSlice(ip) + return a.Unmap(), ok +} diff --git a/app-policy/policyscale/policyscale_test.go b/app-policy/policyscale/policyscale_test.go new file mode 100644 index 00000000000..38b7dcecee3 --- /dev/null +++ b/app-policy/policyscale/policyscale_test.go @@ -0,0 +1,339 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policyscale_test + +import ( + "bytes" + "fmt" + "strings" + "testing" + + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/yaml" + + . "github.com/projectcalico/calico/app-policy/policyscale" + validator "github.com/projectcalico/calico/libcalico-go/lib/validator/v3" +) + +func TestPresetScale(t *testing.T) { + cases := []struct { + name string + spec Spec + ingressRules, egressRules int + ingressPolicies, egressPol int + sets int + }{ + {"baseline", Baseline(), 294 * 68, 0, 294, 0, 3708 + 1}, + {"egress", EgressAllowList(), 0, 301 * 62, 0, 301, 3862}, + {"composite", Composite(), 294 * 68, 301 * 62, 294, 301, 3708 + 1 + 3862}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + fx := Build(c.spec) + if got := fx.Rules(Ingress); got != c.ingressRules { + t.Errorf("ingress rules: got %d, want %d", got, c.ingressRules) + } + if got := fx.Rules(Egress); got != c.egressRules { + t.Errorf("egress rules: got %d, want %d", got, c.egressRules) + } + if got := fx.Policies(Ingress); got != c.ingressPolicies { + t.Errorf("ingress policies: got %d, want %d", got, c.ingressPolicies) + } + if got := fx.Policies(Egress); got != c.egressPol { + t.Errorf("egress policies: got %d, want %d", got, c.egressPol) + } + if got := fx.IPSets(); got != c.sets { + t.Errorf("IP sets: got %d, want %d", got, c.sets) + } + if got := fx.Tiers(); len(got) != 1 || got[0] != "perimeter" { + t.Errorf("tiers: got %v, want [perimeter]", got) + } + store := fx.NewStore() + if got := len(store.PolicyByID); got != c.ingressPolicies+c.egressPol { + t.Errorf("store policies: got %d", got) + } + if got := len(store.IPSetByID); got != c.sets { + t.Errorf("store IP sets: got %d, want %d", got, c.sets) + } + if got := len(fx.Updates()); got != c.sets+c.ingressPolicies+c.egressPol { + t.Errorf("updates: got %d", got) + } + }) + } +} + +func TestBaselineMembersMatchMeasurement(t *testing.T) { + fx := Build(Baseline()) + // The histogram sums to 3,708 sets. Sizes are drawn uniformly within each bucket, so the + // 1,000-9,999 bucket alone contributes ~870k members and the set holds about 1M in all, four + // times the measured ~256k; the measured distribution was skewed towards the small end of each + // bucket. Pin the generated total so that a change to it is deliberate. + if got := fx.IPSetMembers(); got < 950_000 || got > 1_100_000 { + t.Errorf("baseline members: got %d, want about 1M", got) + } +} + +func TestMissingSets(t *testing.T) { + spec := Baseline() + spec.Baseline.MissingIPSets = 8 + fx := Build(spec) + if got := fx.MissingIPSets(); got != 8 { + t.Fatalf("missing sets: got %d, want 8", got) + } + if fx.MissingSetReferences() < 8 { + t.Errorf("missing references: got %d, want at least one per missing set", fx.MissingSetReferences()) + } + if got := len(fx.NewStore().IPSetByID); got != 3709-8 { + t.Errorf("store sets: got %d, want %d", got, 3709-8) + } + // Every resource is still rendered: a cluster never has a missing set. + if got := countKind(fx.Resources(ResourceOptions{}), v3.KindGlobalNetworkSet); got != 3709 { + t.Errorf("rendered sets: got %d, want 3709", got) + } +} + +func TestDeterministic(t *testing.T) { + a, b := render(t, Build(Composite())), render(t, Build(Composite())) + if !bytes.Equal(a, b) { + t.Fatal("two builds of the same spec rendered differently") + } + other := Composite() + other.Seed++ + if bytes.Equal(a, render(t, Build(other))) { + t.Fatal("a different seed rendered the same resources") + } +} + +func TestEgressTargetAndOracle(t *testing.T) { + fx := Build(EgressAllowList()) + tgt := fx.EgressTarget() + if tgt == nil { + t.Fatal("no egress target") + } + numRules := 301 * 62 + if tgt.Ordinal != int(0.65*float64(numRules)) || tgt.RulesWalked != tgt.Ordinal+1 { + t.Errorf("target position: %+v", tgt) + } + want := []Verdict{{Kind: v3.KindGlobalNetworkPolicy, Tier: "perimeter", Policy: tgt.Policy, Index: tgt.RuleIndex, Action: "allow"}} + for name, f := range map[string]*Flow{ + "tail port": NewFlow(SourceIP, DefaultSourcePort, tgt.AddrInCIDR, int(tgt.TailPort)), + "popular port": NewFlow(SourceIP, DefaultSourcePort, tgt.AddrInCIDR, int(tgt.PopularPort)), + "aimed": fx.MatchingFlow(Egress, tgt.Ordinal), + } { + if got := fx.Expect(Egress, f); !equalVerdicts(got, want) { + t.Errorf("%s flow %v: got %v, want %v", name, f, got, want) + } + } + + denied := fx.Expect(Egress, fx.DeniedFlow(Egress)) + wantDenied := []Verdict{{Kind: v3.KindGlobalNetworkPolicy, Tier: "perimeter", Policy: "egress-000", Index: TierDefaultIndex, Action: "deny"}} + if !equalVerdicts(denied, wantDenied) { + t.Errorf("denied flow: got %v, want %v", denied, wantDenied) + } + + // A flow aimed at any rule is matched by that rule or an earlier one, never a later one. + for _, ordinal := range []int{0, 1, 4000, 18661} { + f := fx.MatchingFlow(Egress, ordinal) + got := fx.Expect(Egress, f) + if len(got) == 0 || got[0].Index == TierDefaultIndex { + t.Errorf("flow aimed at rule %d was denied: %v", ordinal, got) + continue + } + if pos := egressOrdinal(got[0]); pos > ordinal { + t.Errorf("flow aimed at rule %d matched later rule %d (%v)", ordinal, pos, got[0]) + } + } +} + +func TestBaselineOracle(t *testing.T) { + fx := Build(Baseline()) + denied := fx.Expect(Ingress, fx.DeniedFlow(Ingress)) + wantDenied := []Verdict{{Kind: v3.KindGlobalNetworkPolicy, Tier: "perimeter", Policy: "policy-000", Index: TierDefaultIndex, Action: "deny"}} + if !equalVerdicts(denied, wantDenied) { + t.Errorf("denied flow: got %v, want %v", denied, wantDenied) + } + + // The set-guarded rules match a flow drawn from their set; the port-guarded ones never do, + // so a flow aimed at one of those is matched by the first port-guarded rule instead. + passes, finals := 0, 0 + for ordinal := 0; ordinal < fx.Rules(Ingress); ordinal += 97 { + f := fx.MatchingFlow(Ingress, ordinal) + got := fx.Expect(Ingress, f) + if len(got) == 0 { + t.Fatalf("empty trace for %v", f) + } + if got[0].Index == TierDefaultIndex { + t.Errorf("flow aimed at rule %d was denied: %v", ordinal, got) + continue + } + if pos := baselineOrdinal(got[0]); pos > ordinal { + t.Errorf("flow aimed at rule %d matched later rule %d (%v)", ordinal, pos, got[0]) + } + switch got[0].Action { + case "pass": + passes++ + // A pass with no further tier and no profiles ends in the profile deny. + if len(got) != 2 || got[1].Kind != v3.KindProfile || got[1].Action != "deny" { + t.Errorf("pass trace for rule %d: %v", ordinal, got) + } + default: + finals++ + if len(got) != 1 { + t.Errorf("final trace for rule %d: %v", ordinal, got) + } + } + } + if passes == 0 { + t.Error("no aimed flow reached a pass rule") + } +} + +func TestSampler(t *testing.T) { + fx := Build(Composite()) + s := fx.NewSampler(1, FlowModel{Direction: Egress, MissFraction: 0.2, RepeatFraction: 0.3}) + seen := map[string]int{} + denied := 0 + for i := 0; i < 2000; i++ { + f := s.Next() + if f.Protocol != ProtocolTCP || f.SrcPort < 32768 { + t.Fatalf("unexpected flow %v", f) + } + key := fmt.Sprintf("%s>%s:%d", f.SrcIP, f.DstIP, f.DstPort) + seen[key]++ + if v := fx.Expect(Egress, f); v[0].Index == TierDefaultIndex { + denied++ + } + } + if denied < 200 || denied > 1000 { + t.Errorf("denied flows: %d of 2000, want roughly 20%% plus their repeats", denied) + } + repeats := 0 + for _, n := range seen { + repeats += n - 1 + } + if repeats < 300 { + t.Errorf("repeated flows: %d of 2000, want roughly 30%%", repeats) + } +} + +func TestResourcesValidateAndRoundTrip(t *testing.T) { + fx := Build(Composite()) + opts := ResourceOptions{Selector: "policyscale.projectcalico.org/target == 'true'"} + objs := fx.Resources(opts) + if got := countKind(objs, v3.KindTier); got != 1 { + t.Errorf("tiers: %d", got) + } + if got := countKind(objs, v3.KindGlobalNetworkPolicy); got != 294+301 { + t.Errorf("policies: %d", got) + } + if got := countKind(objs, v3.KindGlobalNetworkSet); got != 3709+3862 { + t.Errorf("sets: %d", got) + } + + // Validate a sample through the API validator: the tier, the first policies of each direction, + // the target policy, and a few sets including the largest. + validated := 0 + for _, obj := range objs { + validate := false + switch o := obj.(type) { + case *v3.Tier: + validate = true + case *v3.GlobalNetworkPolicy: + validate = strings.HasSuffix(o.Name, "-000") || strings.HasSuffix(o.Name, "-001") || o.Name == "perimeter."+fx.EgressTarget().Policy + case *v3.GlobalNetworkSet: + validate = o.Name == SentinelIPSetID || strings.HasSuffix(o.Name, "-0000") || len(o.Spec.Nets) > 50000 + } + if !validate { + continue + } + if err := validator.Validate(obj); err != nil { + t.Errorf("%T %s: %v", obj, obj.(interface{ GetName() string }).GetName(), err) + } + validated++ + } + if validated < 8 { + t.Errorf("validated only %d objects", validated) + } + + docs := bytes.Split(render(t, fx, opts), []byte("\n---\n")) + if len(docs) != len(objs) { + t.Fatalf("yaml documents: got %d, want %d", len(docs), len(objs)) + } + var gnp v3.GlobalNetworkPolicy + if err := yaml.Unmarshal(docs[1], &gnp); err != nil { + t.Fatal(err) + } + if gnp.Name != "perimeter.policy-000" || gnp.Spec.Tier != "perimeter" || gnp.Spec.Selector != opts.Selector { + t.Errorf("first policy: %+v", gnp.ObjectMeta) + } + if len(gnp.Spec.Types) != 1 || gnp.Spec.Types[0] != v3.PolicyTypeIngress || len(gnp.Spec.Ingress) != 68 { + t.Errorf("first policy rules: types=%v ingress=%d", gnp.Spec.Types, len(gnp.Spec.Ingress)) + } + for i, r := range gnp.Spec.Ingress { + if len(r.Destination.Ports) > 0 && (r.Protocol == nil || r.Protocol.String() != "TCP") { + t.Errorf("rule %d has ports but protocol %v", i, r.Protocol) + } + } +} + +func render(t *testing.T, fx *Fixture, opts ...ResourceOptions) []byte { + t.Helper() + var o ResourceOptions + if len(opts) > 0 { + o = opts[0] + } + var buf bytes.Buffer + if err := fx.WriteYAML(&buf, o); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func countKind(objs []runtime.Object, kind string) int { + n := 0 + for _, o := range objs { + if o.GetObjectKind().GroupVersionKind().Kind == kind { + n++ + } + } + return n +} + +func equalVerdicts(a, b []Verdict) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// egressOrdinal and baselineOrdinal recover a rule's position in the walk from its verdict, using +// the presets' policy naming and rule counts. +func egressOrdinal(v Verdict) int { + var n int + fmt.Sscanf(v.Policy, "egress-%d", &n) + return n*62 + v.Index +} + +func baselineOrdinal(v Verdict) int { + var n int + fmt.Sscanf(v.Policy, "policy-%d", &n) + return n*68 + v.Index +} diff --git a/app-policy/policyscale/resources.go b/app-policy/policyscale/resources.go new file mode 100644 index 00000000000..51514ca6187 --- /dev/null +++ b/app-policy/policyscale/resources.go @@ -0,0 +1,177 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policyscale + +import ( + "fmt" + "io" + + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/yaml" +) + +// ResourceOptions control how a fixture is rendered as Calico resources. +type ResourceOptions struct { + // Selector chooses the endpoints the policies apply to. Default: all(). + Selector string + // SetLabelKey is the label that ties a GlobalNetworkSet to the rules that select it. + // Default: policyscale.projectcalico.org/set. + SetLabelKey string + // TierOrder is the order of the generated tier(s); consecutive tiers count up from it. + // Default 100. + TierOrder float64 +} + +const ( + defaultSelector = "all()" + defaultSetLabelKey = "policyscale.projectcalico.org/set" + defaultTierOrder = 100 + apiVersion = v3.GroupVersionCurrent +) + +func (o *ResourceOptions) defaults() { + if o.Selector == "" { + o.Selector = defaultSelector + } + if o.SetLabelKey == "" { + o.SetLabelKey = defaultSetLabelKey + } + if o.TierOrder == 0 { + o.TierOrder = defaultTierOrder + } +} + +// Resources renders the fixture as the Calico resources that apply the same policy set on a +// cluster: one Tier per generated tier, one GlobalNetworkPolicy per policy (named tier.policy, in +// tier order), and one GlobalNetworkSet per IP set the store holds, labelled so that the rules' +// selectors pick it. Sets marked missing are rendered too: on a cluster nothing is ever "not +// found", and leaving them out would change the verdicts. +// +// Two departures from the proto the engine sees, both forced by the v3 API: a rule with ports +// carries protocol TCP, since ports require a protocol (every generated flow is TCP, so verdicts +// are unchanged), and policy names carry the tier prefix. +func (fx *Fixture) Resources(opts ResourceOptions) []runtime.Object { + opts.defaults() + var out []runtime.Object + + for i, t := range fx.tiers { + order := opts.TierOrder + float64(i) + action := v3.Action(t.defaultAction) + out = append(out, &v3.Tier{ + TypeMeta: metav1.TypeMeta{Kind: v3.KindTier, APIVersion: apiVersion}, + ObjectMeta: metav1.ObjectMeta{Name: t.name}, + Spec: v3.TierSpec{Order: &order, DefaultAction: &action}, + }) + } + + for _, t := range fx.tiers { + policyOrder := float64(0) + for dir, policies := range t.policies { + for _, p := range policies { + gnp := &v3.GlobalNetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: v3.KindGlobalNetworkPolicy, APIVersion: apiVersion}, + ObjectMeta: metav1.ObjectMeta{Name: t.name + "." + p.name}, + } + order := policyOrder + policyOrder++ + gnp.Spec = v3.GlobalNetworkPolicySpec{ + Tier: t.name, + Order: &order, + Selector: opts.Selector, + } + rules := make([]v3.Rule, len(p.rules)) + for i, r := range p.rules { + rules[i] = r.resource(opts.SetLabelKey) + } + if Direction(dir) == Egress { + gnp.Spec.Types = []v3.PolicyType{v3.PolicyTypeEgress} + gnp.Spec.Egress = rules + } else { + gnp.Spec.Types = []v3.PolicyType{v3.PolicyTypeIngress} + gnp.Spec.Ingress = rules + } + out = append(out, gnp) + } + } + } + + for _, id := range fx.setOrder { + s := fx.sets[id] + out = append(out, &v3.GlobalNetworkSet{ + TypeMeta: metav1.TypeMeta{Kind: v3.KindGlobalNetworkSet, APIVersion: apiVersion}, + ObjectMeta: metav1.ObjectMeta{ + Name: id, + Labels: map[string]string{opts.SetLabelKey: id}, + }, + Spec: v3.GlobalNetworkSetSpec{Nets: s.members}, + }) + } + return out +} + +func (r *ruleModel) resource(setLabelKey string) v3.Rule { + rule := v3.Rule{Action: v3.Action(capitalise(r.action))} + if len(r.dstPorts) > 0 { + tcp := numorstring.ProtocolFromString("TCP") + rule.Protocol = &tcp + for _, p := range r.dstPorts { + rule.Destination.Ports = append(rule.Destination.Ports, numorstring.SinglePort(uint16(p))) + } + } + if r.dstNet.IsValid() { + rule.Destination.Nets = []string{r.dstNet.String()} + } + rule.Source.Selector = setSelector(setLabelKey, r.srcSet) + rule.Source.NotSelector = setSelector(setLabelKey, r.notSrcSet) + rule.Destination.Selector = setSelector(setLabelKey, r.dstSet) + rule.Destination.NotSelector = setSelector(setLabelKey, r.notDstSet) + return rule +} + +func setSelector(key, setID string) string { + if setID == "" { + return "" + } + return fmt.Sprintf("%s == '%s'", key, setID) +} + +func capitalise(action string) string { + if action == "" { + return "" + } + return string(action[0]-'a'+'A') + action[1:] +} + +// WriteYAML writes Resources as a multi-document YAML stream, ready for kubectl apply -f. +func (fx *Fixture) WriteYAML(w io.Writer, opts ResourceOptions) error { + for i, obj := range fx.Resources(opts) { + if i > 0 { + if _, err := io.WriteString(w, "---\n"); err != nil { + return err + } + } + b, err := yaml.Marshal(obj) + if err != nil { + return err + } + if _, err := w.Write(b); err != nil { + return err + } + } + return nil +} diff --git a/app-policy/policyscale/spec.go b/app-policy/policyscale/spec.go new file mode 100644 index 00000000000..62be3341933 --- /dev/null +++ b/app-policy/policyscale/spec.go @@ -0,0 +1,179 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policyscale + +// Direction of the rules a policy carries, and of an evaluation. +type Direction int + +const ( + Ingress Direction = iota + Egress +) + +func (d Direction) String() string { + if d == Egress { + return "egress" + } + return "ingress" +} + +// Spec describes a synthetic policy set applied to one endpoint. A nil preset is left out. +type Spec struct { + // Seed drives every random choice. Two Specs that are equal build identical fixtures. + Seed int64 + Baseline *BaselineParams + Egress *EgressParams +} + +// DefaultSeed is the seed the presets use; it is the one the original benchmarks were written +// with, so their numbers stay comparable. +const DefaultSeed = 20200 + +// SizeBucket is one bucket of an IP set size histogram: Sets sets of between MinSize and MaxSize +// members each. +type SizeBucket struct { + Sets, MinSize, MaxSize int +} + +// BaselineParams describes a policy store dominated by "baseline" policies: every policy applies +// to the endpoint, and almost every rule is a Pass. The rules are ingress rules. +type BaselineParams struct { + Tier string + Policies int + RulesPerPolicy int + // DenyRules and AllowRules are spread at random over the rule slots; every other rule is a Pass. + DenyRules int + AllowRules int + // IPSetRefFraction is the fraction of rules that reference an IP set. A referencing rule pairs + // the reference with a negated reference to the sentinel set (which holds the denied flow's + // addresses) so that it reaches the set lookup whatever order the engine evaluates criteria in + // and still misses when the referenced set is absent. Rules with no reference are guarded by a + // destination port no generated flow uses (BaselineGuardPort). + IPSetRefFraction float64 + // MissingIPSets is how many referenced sets to leave out of the store, reproducing the + // "IPSet not found" warning storm seen when the store is out of sync. + MissingIPSets int + // SizeHistogram is the IP set size distribution. Members are unique /32s from 10.0.0.0/9. + SizeHistogram []SizeBucket +} + +// PortWeight is the fraction of egress rules whose destination ports include Port. +type PortWeight struct { + Port int32 + Fraction float64 +} + +// PortCount is the fraction of egress rules that carry Count destination ports. +type PortCount struct { + Count int + Fraction float64 +} + +// EgressParams describes a single-tier destination allow-list: every policy applies to the +// endpoint, every rule is a Pass matching a destination address plus a handful of destination +// ports, and a flow that matches nothing meets the tier default deny. +type EgressParams struct { + Tier string + Policies int + RulesPerPolicy int + // IPSetRuleFraction is the fraction of rules whose destination is an IP set (a selector, once + // it reaches the engine); the rest carry a CIDR. CIDRs are handed out sequentially from + // 10.0.0.0/9 so that no two rules share one. + IPSetRuleFraction float64 + // IPSets and MembersPerSet size the destination sets. Members are unique /32s from + // 10.128.0.0/9, disjoint from the rule CIDRs. + IPSets int + MembersPerSet int + // PortWeights is the head of the destination port distribution; PortsPerRule the spread of + // port counts per rule. Rules top up from a long tail of otherwise-unique ports. + PortWeights []PortWeight + PortsPerRule []PortCount + // TargetDepth, when positive, turns the rule at this fraction of the walk into an Allow with a + // CIDR and a port of its own, so that a flow aimed at it matches nothing earlier. The + // benchmarks use it to measure a match at a known depth; see Fixture.EgressTarget. + TargetDepth float64 +} + +// DefaultBaseline returns the anonymised per-node scale measured in a large production +// deployment: 294 policies of 68 rules, ~24% referencing one of 3,708 IP sets, dominated by tiny +// sets with a long tail of large ones. The measured sets held ~256k members in total; drawn +// uniformly within each bucket the histogram gives about 1M, which the tests pin. +func DefaultBaseline() BaselineParams { + return BaselineParams{ + Tier: "perimeter", + Policies: 294, + RulesPerPolicy: 68, + DenyRules: 10, + AllowRules: 1, + IPSetRefFraction: 0.242, // ~4,838 IP set references across 294*68 rules. + SizeHistogram: []SizeBucket{ + {4, 0, 0}, + {3035, 1, 9}, + {446, 10, 99}, + {64, 100, 999}, + {158, 1000, 9999}, + {1, 54566, 54566}, + }, + } +} + +// DefaultEgress returns the second rule-set shape measured in production: 301 policies of 62 +// egress rules (18,662 in all), 4,644 of 18,675 measured rules with a selector destination, and +// the measured port distribution (443 on 18% of rules, 11001 and 27054 on 12% each, 80 on 8%; +// median 2 ports per rule, mean ~5). The target sits at 65% of the walk. +func DefaultEgress() EgressParams { + return EgressParams{ + Tier: "perimeter", + Policies: 301, + RulesPerPolicy: 62, + IPSetRuleFraction: 0.25, + IPSets: 3862, + MembersPerSet: 3, + PortWeights: []PortWeight{ + {443, 0.182}, + {11001, 0.123}, + {27054, 0.122}, + {80, 0.081}, + }, + PortsPerRule: []PortCount{ + {1, 0.25}, + {2, 0.45}, + {3, 0.15}, + {8, 0.10}, + {20, 0.05}, + }, + TargetDepth: 0.65, + } +} + +// Baseline is the baseline preset on its own. +func Baseline() Spec { + p := DefaultBaseline() + return Spec{Seed: DefaultSeed, Baseline: &p} +} + +// EgressAllowList is the egress allow-list preset on its own. +func EgressAllowList() Spec { + p := DefaultEgress() + return Spec{Seed: DefaultSeed, Egress: &p} +} + +// Composite applies both presets to one endpoint: the baseline set governs its ingress +// evaluations and the allow-list its egress evaluations. This is the reference set the +// PMREQ-954 targets are quoted against. +func Composite() Spec { + b, e := DefaultBaseline(), DefaultEgress() + return Spec{Seed: DefaultSeed, Baseline: &b, Egress: &e} +} diff --git a/app-policy/policyscale/store.go b/app-policy/policyscale/store.go new file mode 100644 index 00000000000..17fbc386c25 --- /dev/null +++ b/app-policy/policyscale/store.go @@ -0,0 +1,97 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policyscale + +import ( + "github.com/projectcalico/calico/app-policy/policystore" + "github.com/projectcalico/calico/felix/proto" + "github.com/projectcalico/calico/felix/types" +) + +// NewStore returns a fresh policy store holding the fixture's policies and the IP sets that are +// not marked missing. Policies are shared between stores built from the same fixture; sets are +// built per store, so a test may mutate them. +func (fx *Fixture) NewStore() *policystore.PolicyStore { + store := policystore.NewPolicyStore() + fx.LoadStore(store) + return store +} + +// LoadStore adds the fixture's policies and IP sets to an existing store, for callers that own +// the store's lifecycle (a policystore.PolicyStoreManager, say). +func (fx *Fixture) LoadStore(store *policystore.PolicyStore) { + for _, id := range fx.setOrder { + s := fx.sets[id] + if s.missing { + continue + } + set := policystore.NewIPSet(proto.IPSetUpdate_NET) + for _, m := range s.members { + set.AddString(m) + } + store.IPSetByID[id] = set + } + for _, t := range fx.tiers { + for _, policies := range t.policies { + for _, p := range policies { + store.PolicyByID[types.ProtoToPolicyID(p.id)] = p.policy + } + } + } +} + +// Updates returns the fixture as the dataplane messages Felix would send to load it into a store +// through PolicyStore.ProcessUpdate: IP sets first, then policies, as Felix orders them. The +// endpoint is not included; callers add it under the ID of their choice. +func (fx *Fixture) Updates() []*proto.ToDataplane { + var updates []*proto.ToDataplane + for _, id := range fx.setOrder { + s := fx.sets[id] + if s.missing { + continue + } + updates = append(updates, &proto.ToDataplane{Payload: &proto.ToDataplane_IpsetUpdate{ + IpsetUpdate: &proto.IPSetUpdate{Id: id, Type: proto.IPSetUpdate_NET, Members: s.members}, + }}) + } + for _, t := range fx.tiers { + for _, policies := range t.policies { + for _, p := range policies { + updates = append(updates, &proto.ToDataplane{Payload: &proto.ToDataplane_ActivePolicyUpdate{ + ActivePolicyUpdate: &proto.ActivePolicyUpdate{Id: p.id, Policy: p.policy}, + }}) + } + } + } + return updates +} + +// Endpoint returns a workload endpoint to which every policy of the fixture applies, with one +// TierInfo per tier carrying the ingress and egress policy lists. Each call returns a new +// endpoint, so a test may prepend a policy without disturbing other users of the fixture. +func (fx *Fixture) Endpoint() *proto.WorkloadEndpoint { + ep := &proto.WorkloadEndpoint{State: "active", Name: "policyscale0"} + for _, t := range fx.tiers { + ti := &proto.TierInfo{Name: t.name, DefaultAction: t.defaultAction} + for _, p := range t.policies[Ingress] { + ti.IngressPolicies = append(ti.IngressPolicies, p.id) + } + for _, p := range t.policies[Egress] { + ti.EgressPolicies = append(ti.EgressPolicies, p.id) + } + ep.Tiers = append(ep.Tiers, ti) + } + return ep +} diff --git a/hack/cmd/policyscale/main.go b/hack/cmd/policyscale/main.go new file mode 100644 index 00000000000..02355f9baad --- /dev/null +++ b/hack/cmd/policyscale/main.go @@ -0,0 +1,98 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// policyscale renders one of the app-policy/policyscale presets as Calico resources, for applying +// the benchmark policy sets to a cluster, or prints flows drawn from its flow model with the +// verdict the engine must reach for each, for driving and checking a node-level run. +// +// go run ./hack/cmd/policyscale -preset composite > composite.yaml +// kubectl label pod flowgen-target policyscale.projectcalico.org/target=true +// kubectl apply -f composite.yaml +// +// go run ./hack/cmd/policyscale -preset composite -flows 1000 -direction egress +package main + +import ( + "bufio" + "flag" + "fmt" + "os" + + "github.com/projectcalico/calico/app-policy/policyscale" +) + +func main() { + var ( + preset = flag.String("preset", "composite", "policy set to render: baseline, egress or composite") + seed = flag.Int64("seed", policyscale.DefaultSeed, "generator seed") + selector = flag.String("selector", "policyscale.projectcalico.org/target == 'true'", "endpoint selector the policies apply to") + out = flag.String("out", "-", "output file, - for stdout") + flows = flag.Int("flows", 0, "instead of resources, print this many flows with their expected verdicts") + direction = flag.String("direction", "egress", "direction of the printed flows: ingress or egress") + miss = flag.Float64("miss-fraction", 0.1, "fraction of printed flows that match no rule") + repeat = flag.Float64("repeat-fraction", 0.5, "fraction of printed flows that repeat an earlier one with a new source port") + ) + flag.Parse() + + var spec policyscale.Spec + switch *preset { + case "baseline": + spec = policyscale.Baseline() + case "egress": + spec = policyscale.EgressAllowList() + case "composite": + spec = policyscale.Composite() + default: + fmt.Fprintf(os.Stderr, "unknown preset %q\n", *preset) + os.Exit(2) + } + spec.Seed = *seed + fx := policyscale.Build(spec) + + w := os.Stdout + if *out != "-" { + f, err := os.Create(*out) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + defer f.Close() + w = f + } + bw := bufio.NewWriter(w) + defer bw.Flush() + + fmt.Fprintf(os.Stderr, "%s: %d ingress rules in %d policies, %d egress rules in %d policies, %d IP sets with %d members\n", + *preset, fx.Rules(policyscale.Ingress), fx.Policies(policyscale.Ingress), + fx.Rules(policyscale.Egress), fx.Policies(policyscale.Egress), fx.IPSets(), fx.IPSetMembers()) + + if *flows > 0 { + dir := policyscale.Egress + if *direction == "ingress" { + dir = policyscale.Ingress + } + s := fx.NewSampler(*seed, policyscale.FlowModel{Direction: dir, MissFraction: *miss, RepeatFraction: *repeat}) + fmt.Fprintln(bw, "protocol\tsrc\tsport\tdst\tdport\texpected") + for i := 0; i < *flows; i++ { + f := s.Next() + fmt.Fprintf(bw, "%d\t%s\t%d\t%s\t%d\t%v\n", f.Protocol, f.SrcIP, f.SrcPort, f.DstIP, f.DstPort, fx.Expect(dir, f)) + } + return + } + + if err := fx.WriteYAML(bw, policyscale.ResourceOptions{Selector: *selector}); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/hack/deps.txt b/hack/deps.txt index c9f25f3e0d9..60cb491acd8 100644 --- a/hack/deps.txt +++ b/hack/deps.txt @@ -111,14 +111,20 @@ sigs.k8s.io/yaml v1.6.0 local:api/pkg/apis/projectcalico/v3 local:api/pkg/defaults local:api/pkg/lib/numorstring +local:app-policy/policyscale +local:app-policy/policystore +local:app-policy/types local:crypto/pkg/tls local:felix/ip +local:felix/proto +local:felix/types local:hack/cmd/calico-selector local:hack/cmd/coalesce-imports local:hack/cmd/deps local:hack/cmd/format-go-file local:hack/cmd/gomodder local:hack/cmd/ipam-hammer +local:hack/cmd/policyscale local:hack/perf/cmd/send-perf-results local:hack/test/spider local:lib/logrusr