diff --git a/app-policy/checker/bench_egress_test.go b/app-policy/checker/bench_egress_test.go index bb37194784f..4a5d3204336 100644 --- a/app-policy/checker/bench_egress_test.go +++ b/app-policy/checker/bench_egress_test.go @@ -15,17 +15,12 @@ package checker // Benchmark for Evaluate() against a single-tier egress allow-list, the second rule-set shape -// measured in production (the first is in bench_test.go). All data is synthetic; the scale -// parameters and distributions below are anonymized measurements from a policy dump. +// measured in production (the first is in bench_test.go). The set is policyscale.EgressAllowList; +// see policyscale.DefaultEgress for the measured distributions it reproduces. // // go test ./app-policy/checker/ -run '^$' -bench BenchmarkEvaluateEgressAllowList \ // -benchmem -benchtime 100x -cpu 1 // -// Shape: one tier, ~301 policies with no selector (so all of them apply to every endpoint), -// ~18,600 egress rules, almost all Pass. The tier is a destination allow-list: match a rule -// and leave the tier, match nothing and the tier default denies. Every rule's source block is -// empty; each matches a destination address plus a handful of destination ports. -// // The three cases below must be read separately, because criterion ordering helps them by // different amounts. A flow whose port is shared by few rules is rejected on the port // comparison almost everywhere and never pays for the address criteria; a flow on a popular @@ -34,66 +29,14 @@ package checker // depth. Averaging the three hides that. import ( - "fmt" - "math/rand" - "net" "testing" - v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" - "github.com/projectcalico/calico/app-policy/policystore" - "github.com/projectcalico/calico/felix/proto" + "github.com/projectcalico/calico/app-policy/policyscale" "github.com/projectcalico/calico/felix/rules" - "github.com/projectcalico/calico/felix/types" -) - -const ( - egressNumPolicies = 301 - egressRulesPerPolicy = 62 // 18,662 rules. - - // Fraction of rules whose destination is a selector, which reaches Dikastes as an IP set - // reference; the rest carry a CIDR. Measured: 4,644 of 18,675. - egressIPSetRuleFraction = 0.25 - - // Where in the walk the rule that matches sits, as a fraction of the whole tier. Real - // flows match at every depth; this picks one representative depth so the two matching - // cases are comparable with each other. - egressTargetDepth = 0.65 - - // The flow. The denied case keeps this destination, which no rule's CIDR contains; the - // matching cases replace it with an address inside the target rule's CIDR. - egressSourceIP = "192.0.2.10" - egressDeniedIP = "198.51.100.20" - egressSourcePort = 45000 ) -// egressPortWeights reproduces the measured head of the port distribution: the fraction of -// rules whose destination ports include each of these. 443 appears in 3,395 of 18,675 rules, -// 11001 in 2,289, 27054 in 2,281 and 80 in 1,505. Rules not drawing a head port get ports from -// a long tail of otherwise-unique values, so a tail port is shared by only a rule or two. -var egressPortWeights = []struct { - port int32 - fraction float64 -}{ - {443, 0.182}, - {11001, 0.123}, - {27054, 0.122}, - {80, 0.081}, -} - -// egressPortsPerRule is the measured spread of destination ports per rule: median 2, mean ~5. -var egressPortsPerRule = []struct { - count int - fraction float64 -}{ - {1, 0.25}, - {2, 0.45}, - {3, 0.15}, - {8, 0.10}, - {20, 0.05}, -} - func BenchmarkEvaluateEgressAllowList(b *testing.B) { // A flow on a port few rules share: rejected on the port comparison nearly everywhere. b.Run("TailPort", func(b *testing.B) { @@ -113,44 +56,34 @@ func BenchmarkEvaluateEgressAllowList(b *testing.B) { // egressCase is what one benchmark case measures: the flow, how many rules the walk is // expected to visit, and whether a rule is expected to match it. type egressCase struct { - flow *MockFlow + flow *policyscale.Flow rulesWalked int matches bool } -// egressCaseFunc builds one case from the fixture's target rule. -type egressCaseFunc func(target egressTarget) egressCase +// egressCaseFunc builds one case from the fixture and its target rule. +type egressCaseFunc func(fx *policyscale.Fixture, target *policyscale.EgressTarget) egressCase -func egressTailPortFlow(target egressTarget) egressCase { +func egressTailPortFlow(_ *policyscale.Fixture, target *policyscale.EgressTarget) egressCase { return egressCase{ - flow: egressFlow(target.addrInCIDR, target.tailPort), - rulesWalked: target.rulesWalked, + flow: policyscale.NewFlow(policyscale.SourceIP, policyscale.DefaultSourcePort, target.AddrInCIDR, int(target.TailPort)), + rulesWalked: target.RulesWalked, matches: true, } } -func egressPopularPortFlow(target egressTarget) egressCase { +func egressPopularPortFlow(_ *policyscale.Fixture, target *policyscale.EgressTarget) egressCase { return egressCase{ - flow: egressFlow(target.addrInCIDR, egressPortWeights[0].port), - rulesWalked: target.rulesWalked, + flow: policyscale.NewFlow(policyscale.SourceIP, policyscale.DefaultSourcePort, target.AddrInCIDR, int(target.PopularPort)), + rulesWalked: target.RulesWalked, matches: true, } } -func egressDeniedFlow(_ egressTarget) egressCase { +func egressDeniedFlow(fx *policyscale.Fixture, _ *policyscale.EgressTarget) egressCase { return egressCase{ - flow: egressFlow(egressDeniedIP, egressPortWeights[0].port), - rulesWalked: egressNumPolicies * egressRulesPerPolicy, - } -} - -func egressFlow(destIP string, destPort int32) *MockFlow { - return &MockFlow{ - SourceIP: net.ParseIP(egressSourceIP), - DestIP: net.ParseIP(destIP), - SourcePort: egressSourcePort, - DestPort: int(destPort), - Protocol: 6, // TCP + flow: fx.DeniedFlow(policyscale.Egress), + rulesWalked: fx.Rules(policyscale.Egress), } } @@ -158,8 +91,9 @@ func benchEvaluateEgressAllowList(b *testing.B, caseFor egressCaseFunc) { _, restoreLogging := withBenchLogging(log.WarnLevel) defer restoreLogging() - store, ep, target := buildEgressAllowListStore() - c := caseFor(target) + fx := policyscale.Build(policyscale.EgressAllowList()) + store, ep, target := fx.NewStore(), fx.Endpoint(), fx.EgressTarget() + c := caseFor(fx, target) // Pre-flight outside the timed loop: prove the walk is the one the case intends, so that // a fixture change cannot silently turn a full walk into an early exit. @@ -168,8 +102,8 @@ func benchEvaluateEgressAllowList(b *testing.B, caseFor egressCaseFunc) { b.Fatalf("evaluation failed: %v", err) } if c.matches { - if len(trace) != 1 || trace[0].Action != rules.RuleActionAllow || trace[0].Index != target.ruleIndex { - b.Fatalf("expected an allow from the target rule at index %d, got %v", target.ruleIndex, trace) + if len(trace) != 1 || trace[0].Action != rules.RuleActionAllow || trace[0].Index != target.RuleIndex { + b.Fatalf("expected an allow from the target rule at index %d, got %v", target.RuleIndex, trace) } } else if len(trace) != 1 || trace[0].Action != rules.RuleActionDeny || trace[0].Index != tierDefaultActionIndex { b.Fatalf("expected a full walk ending in the tier default deny, got %v", trace) @@ -184,125 +118,3 @@ func benchEvaluateEgressAllowList(b *testing.B, caseFor egressCaseFunc) { b.ReportMetric(float64(c.rulesWalked), "rules/op") b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N)/float64(c.rulesWalked), "ns/rule") } - -// egressTarget describes the one rule in the fixture that a matching flow is built to hit. -type egressTarget struct { - ruleIndex int // Index of the rule within its policy, for the trace assertion. - addrInCIDR string // An address inside the rule's destination CIDR. - tailPort int32 // A port on the rule that few other rules share. - rulesWalked int // Rules visited before and including it. -} - -// buildEgressAllowListStore builds the policy store and the endpoint whose single tier applies -// every policy, and returns the target rule a matching flow is aimed at. -// -// Destination CIDRs are handed out sequentially rather than at random so that no rule except -// the target can contain the matching flow's address: a second rule matching earlier would -// shorten the walk and make the case measure something other than what it claims to. -func buildEgressAllowListStore() (*policystore.PolicyStore, *proto.WorkloadEndpoint, egressTarget) { - rng := rand.New(rand.NewSource(20200)) - store := policystore.NewPolicyStore() - setIDs := makeEgressIPSets(store) - - numRules := egressNumPolicies * egressRulesPerPolicy - targetRule := int(float64(numRules) * egressTargetDepth) - - tier := &proto.TierInfo{Name: "perimeter", DefaultAction: "Deny"} - var target egressTarget - nextCIDR := 0 - ruleIdx := 0 - for i := 0; i < egressNumPolicies; i++ { - policyID := &proto.PolicyID{Name: fmt.Sprintf("egress-%03d", i), Kind: v3.KindGlobalNetworkPolicy} - policy := &proto.Policy{Tier: tier.Name} - for j := 0; j < egressRulesPerPolicy; j++ { - // All rules are Pass bar the target: a rule's action is only consulted once it - // matches, so the action mix does not affect the walk. - rule := &proto.Rule{Action: "pass", DstPorts: makeEgressRulePorts(rng)} - if rng.Float64() < egressIPSetRuleFraction { - rule.DstIpSetIds = []string{setIDs[rng.Intn(len(setIDs))]} - } else { - rule.DstNet = []string{fmt.Sprintf("10.%d.%d.0/24", nextCIDR>>8&0xff, nextCIDR&0xff)} - nextCIDR++ - } - if ruleIdx == targetRule { - // Make the target reachable on address and on a port of its own, and give it - // a distinct action so the trace assertion is unambiguous. - rule.Action = "allow" - rule.DstNet = []string{fmt.Sprintf("10.%d.%d.0/24", nextCIDR>>8&0xff, nextCIDR&0xff)} - rule.DstIpSetIds = nil - // Carry both ports the matching cases use, so they differ only in how many - // rules along the way survive the port comparison. - rule.DstPorts = append(rule.DstPorts, - &proto.PortRange{First: egressTailPort, Last: egressTailPort}, - &proto.PortRange{First: egressPortWeights[0].port, Last: egressPortWeights[0].port}, - ) - target = egressTarget{ - ruleIndex: j, - addrInCIDR: fmt.Sprintf("10.%d.%d.7", nextCIDR>>8&0xff, nextCIDR&0xff), - tailPort: egressTailPort, - rulesWalked: ruleIdx + 1, - } - nextCIDR++ - } - policy.OutboundRules = append(policy.OutboundRules, rule) - ruleIdx++ - } - store.PolicyByID[types.ProtoToPolicyID(policyID)] = policy - tier.EgressPolicies = append(tier.EgressPolicies, policyID) - } - - ep := &proto.WorkloadEndpoint{Tiers: []*proto.TierInfo{tier}} - return store, ep, target -} - -// egressTailPort is outside the range makeEgressRulePorts draws from, so only the target rule -// carries it — the extreme of the measured tail, where 90% of ports are on 5 rules or fewer. -const egressTailPort = 60999 - -// makeEgressRulePorts picks a rule's destination ports: the measured head with its measured -// frequency, topped up from a wide tail. -func makeEgressRulePorts(rng *rand.Rand) []*proto.PortRange { - var ports []int32 - for _, w := range egressPortWeights { - if rng.Float64() < w.fraction { - ports = append(ports, w.port) - } - } - for _, n := range egressPortsPerRule { - if rng.Float64() < n.fraction { - for len(ports) < n.count { - ports = append(ports, int32(1024+rng.Intn(55000))) - } - break - } - } - if len(ports) == 0 { - ports = append(ports, int32(1024+rng.Intn(55000))) - } - - ranges := make([]*proto.PortRange, 0, len(ports)) - for _, p := range ports { - ranges = append(ranges, &proto.PortRange{First: p, Last: p}) - } - return ranges -} - -// makeEgressIPSets populates the store with the NET sets the selector-matching rules reference: -// 3,862 sets at a median of 3 members, none containing the flow's addresses. Members are /32s -// from 10.128.0.0/9, kept clear of the rule CIDRs handed out from 10.0.0.0/9. -func makeEgressIPSets(store *policystore.PolicyStore) []string { - const numSets = 3862 - ids := make([]string, 0, numSets) - member := 0 - for i := 0; i < numSets; i++ { - id := fmt.Sprintf("s:egress-%04d", i) - s := policystore.NewIPSet(proto.IPSetUpdate_NET) - for j := 0; j < 3; j++ { - s.AddString(fmt.Sprintf("10.%d.%d.%d/32", 128+(member>>16&0x7f), member>>8&0xff, member&0xff)) - member++ - } - store.IPSetByID[id] = s - ids = append(ids, id) - } - return ids -} diff --git a/app-policy/checker/bench_test.go b/app-policy/checker/bench_test.go index dcb5e510d5d..9a0c74ea357 100644 --- a/app-policy/checker/bench_test.go +++ b/app-policy/checker/bench_test.go @@ -14,12 +14,12 @@ package checker -// Benchmark for Evaluate() at the scale observed in a large production deployment: -// hundreds of "baseline" policies that apply to every endpoint, and thousands of IP -// sets. All data is synthetic; the scale parameters and the IP set size distribution -// are anonymized measurements taken from the deployment's diagnostics. +// Benchmarks for Evaluate() at the scale observed in a large production deployment. The policy +// sets come from app-policy/policyscale, which also renders them as Calico resources and drives +// the collector-level benchmark, so the numbers here, there and on a node describe the same sets. // -// Run with: +// BenchmarkEvaluateBaselinePolicyScale: hundreds of "baseline" policies that apply to every +// endpoint, and thousands of IP sets. Run with: // // go test ./app-policy/checker/ -run '^$' -bench BenchmarkEvaluateBaselinePolicyScale \ // -benchmem -benchtime 100x -cpu 1 @@ -32,12 +32,12 @@ package checker // measured ns/op to judge whether a node is evaluation-bound or pacing on something // else. Note that the benchmark discards log output, so a real deployment pays the // log write on top of the formatting cost measured here. +// +// BenchmarkEvaluateComposite applies both measured shapes to one endpoint: the reference set the +// PMREQ-954 target (10k flows/s per node, so at most 100 µs per evaluation) is quoted against. import ( - "fmt" "io" - "math/rand" - "net" "strings" "sync/atomic" "testing" @@ -46,6 +46,7 @@ import ( v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" log "github.com/sirupsen/logrus" + "github.com/projectcalico/calico/app-policy/policyscale" "github.com/projectcalico/calico/app-policy/policystore" "github.com/projectcalico/calico/felix/calc" "github.com/projectcalico/calico/felix/proto" @@ -54,97 +55,129 @@ import ( "github.com/projectcalico/calico/lib/logrusr" ) -// baselinePolicyScaleParams describes a policy store dominated by "baseline" policies: -// every policy applies to the endpoint, and almost every rule is a Pass. The defaults -// are the anonymized per-node scale measured in a large production deployment. -type baselinePolicyScaleParams struct { - seed int64 - numPolicies int - rulesPerPolicy int - numDenyRules int // Rules with a "deny" action, spread at random; the rest are "pass". - numAllowRules int // As numDenyRules, but "allow". - ipsetRefFraction float64 // Fraction of rules that reference an IP set. - numMissingIPSets int // Referenced IP sets to delete from the store ("IPSet not found" storm). -} - -func defaultBaselinePolicyScaleParams() baselinePolicyScaleParams { - return baselinePolicyScaleParams{ - seed: 20200, - numPolicies: 294, - rulesPerPolicy: 68, - numDenyRules: 10, - numAllowRules: 1, - ipsetRefFraction: 0.242, // ~4,838 IP set references across 294*68 rules. - } -} - -// ipSetSizeHistogram is the per-node IP set size distribution measured in the same -// deployment: 3,708 sets, ~256k members in total, dominated by tiny sets with a long -// tail of large ones. -var ipSetSizeHistogram = []struct{ numSets, minSize, maxSize int }{ - {4, 0, 0}, - {3035, 1, 9}, - {446, 10, 99}, - {64, 100, 999}, - {158, 1000, 9999}, - {1, 54566, 54566}, -} - -// The benchmark flow. TEST-NET addresses; generated IP set members and rule CIDRs all -// come from 10.0.0.0/8, so no rule ever matches on address and the walk covers the -// whole policy set. -const ( - benchSourceIP = "192.0.2.10" - benchDestIP = "198.51.100.20" - benchSourcePort = 45000 - benchDestPort = 8080 - - // benchSentinelIPSetID is a set containing the flow's addresses; see makeScaleRule. - benchSentinelIPSetID = "s:bench-sentinel" -) - // benchTraceSink prevents the compiler from eliminating the Evaluate call. var benchTraceSink []*calc.RuleID func BenchmarkEvaluateBaselinePolicyScale(b *testing.B) { b.Run("AllSetsPresent", func(b *testing.B) { - benchEvaluateBaselinePolicyScale(b, defaultBaselinePolicyScaleParams(), log.WarnLevel, false) + benchEvaluateBaselinePolicyScale(b, policyscale.Baseline(), log.WarnLevel, false) }) b.Run("MissingSets", func(b *testing.B) { - p := defaultBaselinePolicyScaleParams() - p.numMissingIPSets = 8 - benchEvaluateBaselinePolicyScale(b, p, log.WarnLevel, false) + spec := policyscale.Baseline() + spec.Baseline.MissingIPSets = 8 + benchEvaluateBaselinePolicyScale(b, spec, log.WarnLevel, false) }) // As MissingSets but with warnings disabled, to isolate the cost of formatting the // "IPSet not found" warnings. b.Run("MissingSetsLogsOff", func(b *testing.B) { - p := defaultBaselinePolicyScaleParams() - p.numMissingIPSets = 8 - benchEvaluateBaselinePolicyScale(b, p, log.ErrorLevel, false) + spec := policyscale.Baseline() + spec.Baseline.MissingIPSets = 8 + benchEvaluateBaselinePolicyScale(b, spec, log.ErrorLevel, false) }) // Fixed per-Evaluate overhead: the first rule of the first policy matches, so the // walk short-circuits immediately. b.Run("MatchEarly", func(b *testing.B) { - benchEvaluateBaselinePolicyScale(b, defaultBaselinePolicyScaleParams(), log.WarnLevel, true) + benchEvaluateBaselinePolicyScale(b, policyscale.Baseline(), log.WarnLevel, true) }) } -func benchEvaluateBaselinePolicyScale(b *testing.B, p baselinePolicyScaleParams, level log.Level, matchEarly bool) { +// BenchmarkEvaluateComposite measures the reference set in the three cases that matter for the +// collector: an ingress flow that misses every baseline rule, an egress flow that misses every +// allow-list rule, and an egress flow that matches the target rule on a port few rules share. +func BenchmarkEvaluateComposite(b *testing.B) { + _, restoreLogging := withBenchLogging(log.WarnLevel) + defer restoreLogging() + + fx := policyscale.Build(policyscale.Composite()) + store, ep := fx.NewStore(), fx.Endpoint() + target := fx.EgressTarget() + + cases := []struct { + name string + dir rules.RuleDir + flow *policyscale.Flow + walk int + final rules.RuleAction + index int + }{ + {"IngressMissAll", rules.RuleDirIngress, fx.DeniedFlow(policyscale.Ingress), fx.Rules(policyscale.Ingress), rules.RuleActionDeny, tierDefaultActionIndex}, + {"EgressMissAll", rules.RuleDirEgress, fx.DeniedFlow(policyscale.Egress), fx.Rules(policyscale.Egress), rules.RuleActionDeny, tierDefaultActionIndex}, + {"EgressTailPort", rules.RuleDirEgress, policyscale.NewFlow(policyscale.SourceIP, policyscale.DefaultSourcePort, target.AddrInCIDR, int(target.TailPort)), target.RulesWalked, rules.RuleActionAllow, target.RuleIndex}, + } + for _, c := range cases { + b.Run(c.name, func(b *testing.B) { + trace, err := Evaluate(StagedAsEnforced, c.dir, store, ep, c.flow) + if err != nil { + b.Fatalf("evaluation failed: %v", err) + } + if len(trace) != 1 || trace[0].Action != c.final || trace[0].Index != c.index { + b.Fatalf("expected %v at index %d, got %v", c.final, c.index, trace) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + benchTraceSink, _ = Evaluate(StagedAsEnforced, c.dir, store, ep, c.flow) + } + b.StopTimer() + b.ReportMetric(float64(c.walk), "rules/op") + b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N)/float64(c.walk), "ns/rule") + }) + } +} + +// BenchmarkEvaluateVerdictCache measures Evaluate with the verdict cache in front of it, on the +// sampler's flow model: new flows aimed uniformly at the walk, a tenth missing every rule, and a +// fraction repeating an earlier flow on a new source port, which is what the collector sees. The +// sampler's allocation of each flow is inside the timed loop, so allocs/op includes one for it. +func BenchmarkEvaluateVerdictCache(b *testing.B) { + _, restoreLogging := withBenchLogging(log.WarnLevel) + defer restoreLogging() + + fx := policyscale.Build(policyscale.Composite()) + ep := fx.Endpoint() + for _, c := range []struct { + name string + cache bool + repeat float64 + }{ + {"Uncached/Repeat50", false, 0.5}, + {"Cached/Repeat0", true, 0}, + {"Cached/Repeat50", true, 0.5}, + {"Cached/Repeat90", true, 0.9}, + } { + b.Run(c.name, func(b *testing.B) { + store := fx.NewStore() + stats := &policystore.VerdictCacheStats{} + if c.cache { + store.Verdicts = policystore.NewVerdictCache(1<<16, stats) + } + s := fx.NewSampler(1, policyscale.FlowModel{Direction: policyscale.Egress, MissFraction: 0.1, RepeatFraction: c.repeat}) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + benchTraceSink, _ = Evaluate(StagedAsEnforced, rules.RuleDirEgress, store, ep, s.Next()) + } + b.StopTimer() + if c.cache { + total := stats.Hits.Load() + stats.Misses.Load() + b.ReportMetric(float64(stats.Hits.Load())/float64(max(total, 1)), "hit-ratio") + } + }) + } +} + +func benchEvaluateBaselinePolicyScale(b *testing.B, spec policyscale.Spec, level log.Level, matchEarly bool) { logger := log.StandardLogger() counter, restoreLogging := withBenchLogging(level) defer restoreLogging() - store, ep, expectedWarns := buildBaselinePolicyStore(p) + fx := policyscale.Build(spec) + store, ep := fx.NewStore(), fx.Endpoint() + expectedWarns := fx.MissingSetReferences() if matchEarly { addMatchEarlyPolicy(store, ep) } - flow := &MockFlow{ - SourceIP: net.ParseIP(benchSourceIP), - DestIP: net.ParseIP(benchDestIP), - SourcePort: benchSourcePort, - DestPort: benchDestPort, - Protocol: 6, // TCP - } + flow := fx.DeniedFlow(policyscale.Ingress) // Pre-flight outside the timed loop: prove the walk is the intended one and that // the warning count matches the analytic count, so that warnings/op is exact. @@ -171,130 +204,13 @@ func benchEvaluateBaselinePolicyScale(b *testing.B, p baselinePolicyScaleParams, } b.StopTimer() b.ReportMetric(float64(counter.count.Load())/float64(b.N), "warnings/op") - rulesWalked := p.numPolicies * p.rulesPerPolicy + rulesWalked := fx.Rules(policyscale.Ingress) if matchEarly { rulesWalked = 1 } b.ReportMetric(float64(rulesWalked), "rules/op") } -// buildBaselinePolicyStore builds a policy store at the given scale, plus an endpoint -// whose single "perimeter" tier applies every policy. It returns the number of rule -// references to deleted (missing) IP sets, which is exactly the number of "IPSet not -// found" warnings one Evaluate of a non-matching flow emits. -func buildBaselinePolicyStore(p baselinePolicyScaleParams) (*policystore.PolicyStore, *proto.WorkloadEndpoint, int) { - rng := rand.New(rand.NewSource(p.seed)) - store := policystore.NewPolicyStore() - setIDs := makeScaleIPSets(rng, store) - - // Pick the rule slots that get the few non-Pass actions. - numRules := p.numPolicies * p.rulesPerPolicy - specialAction := map[int]string{} - for len(specialAction) < p.numDenyRules { - specialAction[rng.Intn(numRules)] = "deny" - } - for n := 0; n < p.numAllowRules; { - slot := rng.Intn(numRules) - if _, ok := specialAction[slot]; !ok { - specialAction[slot] = "allow" - n++ - } - } - - tier := &proto.TierInfo{Name: "perimeter", DefaultAction: "Deny"} - var referencedIDs []string - refCount := map[string]int{} - ruleIdx := 0 - for i := 0; i < p.numPolicies; i++ { - policyID := &proto.PolicyID{Name: fmt.Sprintf("policy-%03d", i), Kind: v3.KindGlobalNetworkPolicy} - policy := &proto.Policy{Tier: tier.Name} - for j := 0; j < p.rulesPerPolicy; j++ { - action := specialAction[ruleIdx] - if action == "" { - action = "pass" - } - rule, refID := makeScaleRule(rng, setIDs, action, p.ipsetRefFraction) - policy.InboundRules = append(policy.InboundRules, rule) - if refID != "" { - if refCount[refID] == 0 { - referencedIDs = append(referencedIDs, refID) - } - refCount[refID]++ - } - ruleIdx++ - } - store.PolicyByID[types.ProtoToPolicyID(policyID)] = policy - tier.IngressPolicies = append(tier.IngressPolicies, policyID) - } - - // Delete some referenced sets to reproduce the "IPSet not found" storm. - rng.Shuffle(len(referencedIDs), func(a, b int) { - referencedIDs[a], referencedIDs[b] = referencedIDs[b], referencedIDs[a] - }) - expectedWarns := 0 - for _, id := range referencedIDs[:p.numMissingIPSets] { - delete(store.IPSetByID, id) - expectedWarns += refCount[id] - } - - ep := &proto.WorkloadEndpoint{Tiers: []*proto.TierInfo{tier}} - return store, ep, expectedWarns -} - -// makeScaleIPSets populates the store with NET-type IP sets (the dominant type in the -// measured deployment: selector and networkset derived sets) following the measured -// size histogram. Members are unique /32s from 10.0.0.0/8. -func makeScaleIPSets(rng *rand.Rand, store *policystore.PolicyStore) []string { - sentinel := policystore.NewIPSet(proto.IPSetUpdate_NET) - sentinel.AddString(benchSourceIP + "/32") - sentinel.AddString(benchDestIP + "/32") - store.IPSetByID[benchSentinelIPSetID] = sentinel - - var ids []string - member := 0 - for _, bucket := range ipSetSizeHistogram { - for i := 0; i < bucket.numSets; i++ { - id := fmt.Sprintf("s:bench-%04d", len(ids)) - s := policystore.NewIPSet(proto.IPSetUpdate_NET) - size := bucket.minSize + rng.Intn(bucket.maxSize-bucket.minSize+1) - for j := 0; j < size; j++ { - s.AddString(fmt.Sprintf("10.%d.%d.%d/32", member>>16&0xff, member>>8&0xff, member&0xff)) - member++ - } - store.IPSetByID[id] = s - ids = append(ids, id) - } - } - return ids -} - -// makeScaleRule builds a rule that never matches the benchmark flow. It returns the -// referenced IP set ID, or "" for a rule with no reference. -// -// A rule that references an IP set must reach the set lookup whatever order match() -// evaluates criteria in, and must still miss when the referenced set is absent from -// the store (an absent set is skipped rather than treated as a non-match). Both hold -// by pairing the reference with a negated reference to benchSentinelIPSetID, which -// contains the flow's addresses: if the referenced set is present the lookup misses -// and evaluation stops there; if it is absent the sentinel makes the rule miss. -func makeScaleRule(rng *rand.Rand, setIDs []string, action string, refFraction float64) (*proto.Rule, string) { - rule := &proto.Rule{Action: action} - if rng.Float64() < refFraction { - id := setIDs[rng.Intn(len(setIDs))] - if rng.Intn(2) == 0 { - rule.SrcIpSetIds = []string{id} - rule.NotSrcIpSetIds = []string{benchSentinelIPSetID} - } else { - rule.DstIpSetIds = []string{id} - rule.NotDstIpSetIds = []string{benchSentinelIPSetID} - } - return rule, id - } - // No IP set reference: guard with a non-matching destination port. - rule.DstPorts = []*proto.PortRange{{First: 65001, Last: 65001}} - return rule, "" -} - // addMatchEarlyPolicy prepends a policy whose first rule matches any flow. func addMatchEarlyPolicy(store *policystore.PolicyStore, ep *proto.WorkloadEndpoint) { policyID := &proto.PolicyID{Name: "policy-match-early", Kind: v3.KindGlobalNetworkPolicy} diff --git a/app-policy/checker/check.go b/app-policy/checker/check.go index c4c68767277..f3276259323 100644 --- a/app-policy/checker/check.go +++ b/app-policy/checker/check.go @@ -123,6 +123,23 @@ const ( // the caller should hold on to whatever trace it already had: an empty trace would say the flow has // no policy, which is a stronger claim than "we could not work it out". func Evaluate(scope PolicyScope, dir rules.RuleDir, store *policystore.PolicyStore, ep *proto.WorkloadEndpoint, flow Flow) ([]*calc.RuleID, error) { + if store != nil && store.Verdicts != nil { + if key, ok := verdictKey(store, scope, dir, ep, flow); ok { + if cached, hit := store.Verdicts.Lookup(store.Generation, key); hit { + return cached.([]*calc.RuleID), nil + } + trace, err := evaluate(scope, dir, store, ep, flow) + if err == nil { + store.Verdicts.Store(store.Generation, key, trace) + } + return trace, err + } + } + return evaluate(scope, dir, store, ep, flow) +} + +// evaluate walks the policies; Evaluate is the cache in front of it. +func evaluate(scope PolicyScope, dir rules.RuleDir, store *policystore.PolicyStore, ep *proto.WorkloadEndpoint, flow Flow) ([]*calc.RuleID, error) { s, trace := checkTiers(scope, store, ep, dir, flow) if s.Code == INTERNAL || s.Code == INVALID_ARGUMENT { // The evaluation stopped part way through, so the trace stops short of a verdict. Drop it diff --git a/app-policy/checker/differential_test.go b/app-policy/checker/differential_test.go new file mode 100644 index 00000000000..361c1a5f086 --- /dev/null +++ b/app-policy/checker/differential_test.go @@ -0,0 +1,442 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package checker + +// Differential tests for Evaluate. +// +// Two evaluators are run over the same corpus and must agree on every trace. Today the second +// evaluator is the policyscale oracle, which answers from the generator's model of each rule +// rather than from the proto the engine reads. A change that adds a second implementation of the +// walk (compiled policies, a verdict cache, evaluation on another goroutine) runs it through the +// same harness against Evaluate, and through the named cases below, before it is switched on. + +import ( + "fmt" + "testing" + + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/app-policy/policyscale" + "github.com/projectcalico/calico/app-policy/policystore" + "github.com/projectcalico/calico/felix/calc" + "github.com/projectcalico/calico/felix/proto" + "github.com/projectcalico/calico/felix/rules" + "github.com/projectcalico/calico/felix/types" +) + +// evaluator is the shape of Evaluate, so that another implementation can be run through the +// same harness. +type evaluator func(scope PolicyScope, dir rules.RuleDir, store *policystore.PolicyStore, ep *proto.WorkloadEndpoint, flow Flow) ([]*calc.RuleID, error) + +// corpusFlow is one evaluation of the differential corpus. +type corpusFlow struct { + name string + dir rules.RuleDir + flow *policyscale.Flow +} + +func TestEvaluateAgreesWithOracle(t *testing.T) { + _, restoreLogging := withBenchLogging(log.ErrorLevel) + defer restoreLogging() + + missing := policyscale.Baseline() + missing.Baseline.MissingIPSets = 8 + + for _, c := range []struct { + name string + spec policyscale.Spec + }{ + {"baseline", policyscale.Baseline()}, + {"baseline with missing sets", missing}, + {"egress allow-list", policyscale.EgressAllowList()}, + {"composite", policyscale.Composite()}, + } { + t.Run(c.name, func(t *testing.T) { + fx := policyscale.Build(c.spec) + store, ep := fx.NewStore(), fx.Endpoint() + corpus := differentialCorpus(fx, 7, 250, 257) + for _, scope := range []PolicyScope{StagedAsEnforced, EnforcedOnly} { + assertEquivalent(t, fmt.Sprintf("%s scope %d", c.name, scope), scope, Evaluate, oracleEvaluator(fx), store, ep, corpus) + } + }) + } +} + +// differentialCorpus draws the flows a fixture is checked with, in every direction it has rules +// for: the denied flow, a flow aimed at every stride-th rule, and sampled flows following a flow +// model. Aimed flows are also replayed as UDP (no generated rule names a protocol, so the verdict +// must not change), and a few with an invalid protocol and with a nil address. +func differentialCorpus(fx *policyscale.Fixture, seed int64, sampled, stride int) []corpusFlow { + var corpus []corpusFlow + for _, dir := range []policyscale.Direction{policyscale.Ingress, policyscale.Egress} { + rulesInDir := fx.Rules(dir) + if rulesInDir == 0 { + continue + } + rd := ruleDir(dir) + add := func(name string, f *policyscale.Flow) { + corpus = append(corpus, corpusFlow{name: fmt.Sprintf("%s %s %v", dir, name, f), dir: rd, flow: f}) + } + add("denied", fx.DeniedFlow(dir)) + for ordinal := 0; ordinal < rulesInDir; ordinal += stride { + f := fx.MatchingFlow(dir, ordinal) + add(fmt.Sprintf("aimed at %d", ordinal), f) + udp := *f + udp.Protocol = 17 + add(fmt.Sprintf("aimed at %d as UDP", ordinal), &udp) + if ordinal%(stride*8) == 0 { + invalid := *f + invalid.Protocol = 256 + add(fmt.Sprintf("aimed at %d with protocol 256", ordinal), &invalid) + noSrc := *f + noSrc.SrcIP = nil + add(fmt.Sprintf("aimed at %d with nil source", ordinal), &noSrc) + noDst := *f + noDst.DstIP = nil + add(fmt.Sprintf("aimed at %d with nil destination", ordinal), &noDst) + } + } + s := fx.NewSampler(seed, policyscale.FlowModel{Direction: dir, MissFraction: 0.1, RepeatFraction: 0.3}) + for i := 0; i < sampled; i++ { + add(fmt.Sprintf("sampled %d", i), s.Next()) + } + } + return corpus +} + +// assertEquivalent evaluates every corpus flow with both evaluators and reports each +// disagreement, up to a limit. The two must agree on the trace and on whether the evaluation +// failed. +func assertEquivalent(t *testing.T, name string, scope PolicyScope, a, b evaluator, store *policystore.PolicyStore, ep *proto.WorkloadEndpoint, corpus []corpusFlow) { + t.Helper() + const maxReported = 10 + mismatches := 0 + for _, c := range corpus { + traceA, errA := a(scope, c.dir, store, ep, c.flow) + traceB, errB := b(scope, c.dir, store, ep, c.flow) + if (errA != nil) != (errB != nil) || !sameTrace(traceA, traceB) { + mismatches++ + if mismatches <= maxReported { + t.Errorf("%s: %s:\n a: %v (err %v)\n b: %v (err %v)", name, c.name, formatTrace(traceA), errA, formatTrace(traceB), errB) + } + } + } + if mismatches > maxReported { + t.Errorf("%s: %d mismatches in %d flows (%d shown)", name, mismatches, len(corpus), maxReported) + } +} + +// oracleEvaluator answers from the fixture's model. Only flows the fixture generated are meaningful +// to it, and it ignores the scope: the generated sets hold no staged policies. +func oracleEvaluator(fx *policyscale.Fixture) evaluator { + return func(_ PolicyScope, dir rules.RuleDir, _ *policystore.PolicyStore, _ *proto.WorkloadEndpoint, flow Flow) ([]*calc.RuleID, error) { + var trace []*calc.RuleID + for _, v := range fx.Expect(policyDir(dir), flow.(*policyscale.Flow)) { + trace = append(trace, calc.NewRuleID(v.Kind, v.Tier, v.Policy, "", v.Index, dir, actionFromOracle(v.Action))) + } + return trace, nil + } +} + +func actionFromOracle(action string) rules.RuleAction { + switch action { + case "allow": + return rules.RuleActionAllow + case "deny": + return rules.RuleActionDeny + case "pass": + return rules.RuleActionPass + } + panic("unexpected oracle action " + action) +} + +func ruleDir(d policyscale.Direction) rules.RuleDir { + if d == policyscale.Egress { + return rules.RuleDirEgress + } + return rules.RuleDirIngress +} + +func policyDir(d rules.RuleDir) policyscale.Direction { + if d == rules.RuleDirEgress { + return policyscale.Egress + } + return policyscale.Ingress +} + +func sameTrace(a, b []*calc.RuleID) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if !sameRuleID(a[i], b[i]) { + return false + } + } + return true +} + +func sameRuleID(a, b *calc.RuleID) bool { + if a == nil || b == nil { + return a == b + } + return a.Kind == b.Kind && a.Tier == b.Tier && a.Name == b.Name && a.Namespace == b.Namespace && + a.Index == b.Index && a.Direction == b.Direction && a.Action == b.Action +} + +func formatTrace(trace []*calc.RuleID) string { + s := "[" + for i, r := range trace { + if i > 0 { + s += " " + } + if r == nil { + s += "" + continue + } + s += fmt.Sprintf("%s/%s/%s[%d]=%v", r.Kind, r.Tier, r.Name, r.Index, r.Action) + } + return s + "]" +} + +// namedCase is one hand-built evaluation with a known answer: the edge cases the generated corpus +// cannot reach. Every evaluator must reproduce the answer. +type namedCase struct { + name string + scope PolicyScope + dir rules.RuleDir + store *policystore.PolicyStore + ep *proto.WorkloadEndpoint + flow Flow + want []*calc.RuleID + wantErr bool +} + +func TestEvaluateNamedCases(t *testing.T) { + _, restoreLogging := withBenchLogging(log.ErrorLevel) + defer restoreLogging() + runNamedCases(t, Evaluate) +} + +// runNamedCases checks an evaluator against every named case. +func runNamedCases(t *testing.T, eval evaluator) { + t.Helper() + for _, c := range namedCases() { + t.Run(c.name, func(t *testing.T) { + got, err := eval(c.scope, c.dir, c.store, c.ep, c.flow) + if c.wantErr { + if err == nil { + t.Fatalf("expected an error, got trace %v", formatTrace(got)) + } + if got != nil { + t.Fatalf("expected no trace with the error, got %v", formatTrace(got)) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !sameTrace(got, c.want) { + t.Fatalf("got %v, want %v", formatTrace(got), formatTrace(c.want)) + } + }) + } +} + +func namedCases() []namedCase { + const ( + tier = "t1" + setA = "set-a" + absent = "set-absent" + npSet = "named-port-set" + ipPort = "ip-port-set" + peerIP = "10.0.0.5" + otherIP = "10.0.0.6" + ) + ingress := rules.RuleDirIngress + tcp := func(src string, dst string, dport int) *policyscale.Flow { + return policyscale.NewFlow(src, 40000, dst, dport) + } + gnp := func(kind, name, tier string, rs ...*proto.Rule) (*proto.PolicyID, *proto.Policy) { + return &proto.PolicyID{Name: name, Kind: kind}, &proto.Policy{Tier: tier, InboundRules: rs} + } + allow := func(r *proto.Rule) *proto.Rule { r.Action = "allow"; return r } + deny := func(r *proto.Rule) *proto.Rule { r.Action = "deny"; return r } + pass := func(r *proto.Rule) *proto.Rule { r.Action = "pass"; return r } + ruleID := func(kind, name string, index int, action rules.RuleAction) *calc.RuleID { + return calc.NewRuleID(kind, tier, name, "", index, ingress, action) + } + profileDeny := calc.NewRuleID(v3.KindProfile, profileStr, profileStr, "", tierDefaultActionIndex, ingress, rules.RuleActionDeny) + + // build returns a store and endpoint with one tier holding the given policies, in order. + build := func(defaultAction string, policies ...func() (*proto.PolicyID, *proto.Policy)) (*policystore.PolicyStore, *proto.WorkloadEndpoint) { + store := policystore.NewPolicyStore() + a := policystore.NewIPSet(proto.IPSetUpdate_NET) + a.AddString(peerIP + "/32") + store.IPSetByID[setA] = a + np := policystore.NewIPSet(proto.IPSetUpdate_IP_AND_PORT) + np.AddString(ipProtoPortKey(peerIP, 6, 8080)) + store.IPSetByID[npSet] = np + ipp := policystore.NewIPSet(proto.IPSetUpdate_IP_AND_PORT) + ipp.AddString(ipProtoPortKey(peerIP, 6, 8080)) + store.IPSetByID[ipPort] = ipp + + ti := &proto.TierInfo{Name: tier, DefaultAction: defaultAction} + for _, p := range policies { + id, policy := p() + store.PolicyByID[types.ProtoToPolicyID(id)] = policy + ti.IngressPolicies = append(ti.IngressPolicies, id) + } + return store, &proto.WorkloadEndpoint{Tiers: []*proto.TierInfo{ti}} + } + policy := func(name string, rs ...*proto.Rule) func() (*proto.PolicyID, *proto.Policy) { + return func() (*proto.PolicyID, *proto.Policy) { return gnp(v3.KindGlobalNetworkPolicy, name, tier, rs...) } + } + stagedPolicy := func(name string, rs ...*proto.Rule) func() (*proto.PolicyID, *proto.Policy) { + return func() (*proto.PolicyID, *proto.Policy) { + return gnp(v3.KindStagedGlobalNetworkPolicy, name, tier, rs...) + } + } + + var cases []namedCase + add := func(name string, scope PolicyScope, store *policystore.PolicyStore, ep *proto.WorkloadEndpoint, flow Flow, want ...*calc.RuleID) { + cases = append(cases, namedCase{name: name, scope: scope, dir: ingress, store: store, ep: ep, flow: flow, want: want}) + } + + // Nil addresses: a positive set or CIDR reference never matches them, a negated one always does. + { + store, ep := build("Deny", + policy("p", allow(&proto.Rule{SrcIpSetIds: []string{setA}}), allow(&proto.Rule{SrcNet: []string{"10.0.0.0/8"}}), allow(&proto.Rule{NotSrcIpSetIds: []string{setA}, NotSrcNet: []string{"10.0.0.0/8"}}))) + add("nil source address", StagedAsEnforced, store, ep, &policyscale.Flow{DstIP: tcp(peerIP, peerIP, 80).DstIP, SrcPort: 1, DstPort: 80, Protocol: 6}, + ruleID(v3.KindGlobalNetworkPolicy, "p", 2, rules.RuleActionAllow)) + } + + // Named ports resolve through the IP+port set on the destination; a negated named port is + // the complement. + { + store, ep := build("Deny", policy("p", allow(&proto.Rule{DstNamedPortIpSetIds: []string{npSet}}))) + add("named port on the flow's port", StagedAsEnforced, store, ep, tcp(otherIP, peerIP, 8080), ruleID(v3.KindGlobalNetworkPolicy, "p", 0, rules.RuleActionAllow)) + add("named port on another port", StagedAsEnforced, store, ep, tcp(otherIP, peerIP, 8081), ruleID(v3.KindGlobalNetworkPolicy, "p", tierDefaultActionIndex, rules.RuleActionDeny)) + store, ep = build("Deny", policy("p", allow(&proto.Rule{NotDstNamedPortIpSetIds: []string{npSet}}))) + add("negated named port on the flow's port", StagedAsEnforced, store, ep, tcp(otherIP, peerIP, 8080), ruleID(v3.KindGlobalNetworkPolicy, "p", tierDefaultActionIndex, rules.RuleActionDeny)) + add("negated named port on another port", StagedAsEnforced, store, ep, tcp(otherIP, peerIP, 8081), ruleID(v3.KindGlobalNetworkPolicy, "p", 0, rules.RuleActionAllow)) + } + + // A numeric port range is tried before the named-port sets and matches on its own. + { + store, ep := build("Deny", policy("p", allow(&proto.Rule{DstPorts: []*proto.PortRange{{First: 8000, Last: 8100}}, DstNamedPortIpSetIds: []string{npSet}}))) + add("port range or named port", StagedAsEnforced, store, ep, tcp(otherIP, otherIP, 8050), ruleID(v3.KindGlobalNetworkPolicy, "p", 0, rules.RuleActionAllow)) + } + + // Negated IP sets, and IP+port sets on the destination. + { + store, ep := build("Deny", policy("p", allow(&proto.Rule{NotDstIpSetIds: []string{setA}}))) + add("negated set holding the destination", StagedAsEnforced, store, ep, tcp(otherIP, peerIP, 80), ruleID(v3.KindGlobalNetworkPolicy, "p", tierDefaultActionIndex, rules.RuleActionDeny)) + add("negated set not holding the destination", StagedAsEnforced, store, ep, tcp(peerIP, otherIP, 80), ruleID(v3.KindGlobalNetworkPolicy, "p", 0, rules.RuleActionAllow)) + store, ep = build("Deny", policy("p", allow(&proto.Rule{DstIpPortSetIds: []string{ipPort}}))) + add("IP+port set on the flow's port", StagedAsEnforced, store, ep, tcp(otherIP, peerIP, 8080), ruleID(v3.KindGlobalNetworkPolicy, "p", 0, rules.RuleActionAllow)) + add("IP+port set on another port", StagedAsEnforced, store, ep, tcp(otherIP, peerIP, 8081), ruleID(v3.KindGlobalNetworkPolicy, "p", tierDefaultActionIndex, rules.RuleActionDeny)) + } + + // A reference to a set the store does not hold is skipped, negated or not. + { + store, ep := build("Deny", policy("p", allow(&proto.Rule{DstIpSetIds: []string{absent}}))) + add("missing set is skipped", StagedAsEnforced, store, ep, tcp(otherIP, otherIP, 80), ruleID(v3.KindGlobalNetworkPolicy, "p", 0, rules.RuleActionAllow)) + store, ep = build("Deny", policy("p", allow(&proto.Rule{NotDstIpSetIds: []string{absent}}))) + add("missing negated set is skipped", StagedAsEnforced, store, ep, tcp(otherIP, peerIP, 80), ruleID(v3.KindGlobalNetworkPolicy, "p", 0, rules.RuleActionAllow)) + } + + // Protocols: out of range matches nothing, even a rule with no protocol; by name and by number; + // negated. + { + store, ep := build("Deny", policy("p", allow(&proto.Rule{}))) + for _, p := range []int{0, 256, -1} { + f := tcp(otherIP, otherIP, 80) + f.Protocol = p + add(fmt.Sprintf("protocol %d matches nothing", p), StagedAsEnforced, store, ep, f, ruleID(v3.KindGlobalNetworkPolicy, "p", tierDefaultActionIndex, rules.RuleActionDeny)) + } + store, ep = build("Deny", policy("p", + allow(&proto.Rule{Protocol: &proto.Protocol{NumberOrName: &proto.Protocol_Name{Name: "UDP"}}}), + allow(&proto.Rule{Protocol: &proto.Protocol{NumberOrName: &proto.Protocol_Number{Number: 132}}}), + allow(&proto.Rule{NotProtocol: &proto.Protocol{NumberOrName: &proto.Protocol_Name{Name: "tcp"}}}), + allow(&proto.Rule{Protocol: &proto.Protocol{NumberOrName: &proto.Protocol_Name{Name: "TCP"}}}))) + add("protocol by name and number", StagedAsEnforced, store, ep, tcp(otherIP, otherIP, 80), ruleID(v3.KindGlobalNetworkPolicy, "p", 3, rules.RuleActionAllow)) + udp := tcp(otherIP, otherIP, 80) + udp.Protocol = 17 + add("protocol by name matches UDP", StagedAsEnforced, store, ep, udp, ruleID(v3.KindGlobalNetworkPolicy, "p", 0, rules.RuleActionAllow)) + sctp := tcp(otherIP, otherIP, 80) + sctp.Protocol = 132 + add("protocol by number matches SCTP", StagedAsEnforced, store, ep, sctp, ruleID(v3.KindGlobalNetworkPolicy, "p", 1, rules.RuleActionAllow)) + } + + // Staged policies: out of scope for the enforced verdict, in scope for the pending one. A tier + // whose policies are all staged contributes nothing to the enforced verdict, default included. + { + store, ep := build("Deny", stagedPolicy("s", allow(&proto.Rule{}))) + add("staged-only tier, enforced scope", EnforcedOnly, store, ep, tcp(otherIP, otherIP, 80), profileDeny) + add("staged-only tier, pending scope", StagedAsEnforced, store, ep, tcp(otherIP, otherIP, 80), ruleID(v3.KindStagedGlobalNetworkPolicy, "s", 0, rules.RuleActionAllow)) + store, ep = build("Deny", policy("e", allow(&proto.Rule{DstPorts: []*proto.PortRange{{First: 1, Last: 1}}})), stagedPolicy("s", allow(&proto.Rule{}))) + add("enforced then staged, enforced scope", EnforcedOnly, store, ep, tcp(otherIP, otherIP, 80), ruleID(v3.KindGlobalNetworkPolicy, "e", tierDefaultActionIndex, rules.RuleActionDeny)) + add("enforced then staged, pending scope", StagedAsEnforced, store, ep, tcp(otherIP, otherIP, 80), ruleID(v3.KindStagedGlobalNetworkPolicy, "s", 0, rules.RuleActionAllow)) + } + + // An L4 flow carries no HTTP attributes, so HTTP criteria match it. + { + store, ep := build("Deny", policy("p", allow(&proto.Rule{HttpMatch: &proto.HTTPMatch{Methods: []string{"GET"}, Paths: []*proto.HTTPMatch_PathMatch{{PathMatch: &proto.HTTPMatch_PathMatch_Exact{Exact: "/x"}}}}}))) + add("HTTP rule against an L4 flow", StagedAsEnforced, store, ep, tcp(otherIP, otherIP, 80), ruleID(v3.KindGlobalNetworkPolicy, "p", 0, rules.RuleActionAllow)) + } + + // A Log rule matches and evaluation continues. + { + store, ep := build("Deny", policy("p", &proto.Rule{Action: "log"}, deny(&proto.Rule{}))) + add("log rule continues", StagedAsEnforced, store, ep, tcp(otherIP, otherIP, 80), ruleID(v3.KindGlobalNetworkPolicy, "p", 1, rules.RuleActionDeny)) + } + + // Pass leaves the tier: the trace carries the pass, then the next tier's verdict; with no next + // tier and no profiles, the profile deny. + { + store, ep := build("Deny", policy("p", pass(&proto.Rule{}))) + add("pass with nothing after it", StagedAsEnforced, store, ep, tcp(otherIP, otherIP, 80), ruleID(v3.KindGlobalNetworkPolicy, "p", 0, rules.RuleActionPass), profileDeny) + store, ep = build("Deny", policy("p", pass(&proto.Rule{}))) + t2 := &proto.TierInfo{Name: "t2", DefaultAction: "Deny"} + id, pol := gnp(v3.KindGlobalNetworkPolicy, "q", "t2", deny(&proto.Rule{})) + store.PolicyByID[types.ProtoToPolicyID(id)] = pol + t2.IngressPolicies = []*proto.PolicyID{id} + ep.Tiers = append(ep.Tiers, t2) + add("pass then next tier", StagedAsEnforced, store, ep, tcp(otherIP, otherIP, 80), + ruleID(v3.KindGlobalNetworkPolicy, "p", 0, rules.RuleActionPass), + calc.NewRuleID(v3.KindGlobalNetworkPolicy, "t2", "q", "", 0, ingress, rules.RuleActionDeny)) + } + + // Tier default Pass continues to the profiles; a profile allow ends there. + { + store, ep := build("Pass", policy("p", allow(&proto.Rule{DstPorts: []*proto.PortRange{{First: 1, Last: 1}}}))) + store.ProfileByID[types.ProtoToProfileID(&proto.ProfileID{Name: "prof"})] = &proto.Profile{InboundRules: []*proto.Rule{allow(&proto.Rule{})}} + ep.ProfileIds = []string{"prof"} + add("tier default pass then profile", StagedAsEnforced, store, ep, tcp(otherIP, otherIP, 80), + ruleID(v3.KindGlobalNetworkPolicy, "p", tierDefaultActionIndex, rules.RuleActionPass), + calc.NewRuleID(v3.KindProfile, profileStr, "prof", "", 0, ingress, rules.RuleActionAllow)) + } + + // A policy the endpoint names but the store lacks fails the evaluation rather than skipping it. + { + store, ep := build("Deny", policy("p", allow(&proto.Rule{}))) + ep.Tiers[0].IngressPolicies = append([]*proto.PolicyID{{Name: "ghost", Kind: v3.KindGlobalNetworkPolicy}}, ep.Tiers[0].IngressPolicies...) + cases = append(cases, namedCase{name: "missing policy fails", scope: StagedAsEnforced, dir: ingress, store: store, ep: ep, flow: tcp(otherIP, otherIP, 80), wantErr: true}) + } + + return cases +} diff --git a/app-policy/checker/verdictcache.go b/app-policy/checker/verdictcache.go new file mode 100644 index 00000000000..40960e4b695 --- /dev/null +++ b/app-policy/checker/verdictcache.go @@ -0,0 +1,114 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package checker + +import ( + "github.com/projectcalico/calico/app-policy/policystore" + "github.com/projectcalico/calico/felix/proto" + "github.com/projectcalico/calico/felix/rules" + ftypes "github.com/projectcalico/calico/felix/types" + "github.com/projectcalico/calico/libcalico-go/lib/backend/model" +) + +// verdictKey decides whether a flow's verdict may be served from the store's cache and builds its +// key. +// +// A verdict is a function of the endpoint's applicable rules, the store's IP sets and the flow. +// The first two are covered by the store generation the cache is bound to; the key covers the +// flow. For a flow with no L7 attributes and no peer identity, the criteria in match() reduce to +// protocol, source and destination address, destination port, and source port; identity and HTTP +// criteria match such a flow whatever the rule says. So the key is those fields, with the source +// port included only when a rule that can apply to the endpoint looks at it, which is decided +// once per endpoint and generation. +// +// Flows that carry identity or HTTP attributes (Dikastes requests) are not cached: their verdict +// also depends on the peer's service account and namespace labels. +func verdictKey(store *policystore.PolicyStore, scope PolicyScope, dir rules.RuleDir, ep *proto.WorkloadEndpoint, flow Flow) (policystore.VerdictKey, bool) { + if ep == nil || !isL4Only(flow) { + return policystore.VerdictKey{}, false + } + src, dst := flow.GetSourceIP().To16(), flow.GetDestIP().To16() + if src == nil || dst == nil { + return policystore.VerdictKey{}, false + } + key := policystore.VerdictKey{ + Endpoint: ep, + Scope: int8(scope), + Direction: int8(dir), + Protocol: int32(flow.GetProtocol()), + SrcPort: -1, + DstPort: int32(flow.GetDestPort()), + } + copy(key.SrcIP[:], src) + copy(key.DstIP[:], dst) + usesSrcPort := store.Verdicts.EndpointFlag(store.Generation, ep, int8(scope), int8(dir), func() bool { + return endpointMatchesSourcePorts(store, ep, scope, dir) + }) + if usesSrcPort { + key.SrcPort = int32(flow.GetSourcePort()) + } + return key, true +} + +// isL4Only reports whether the flow carries nothing but its L3/L4 header, as flows from the +// collector do. +func isL4Only(flow Flow) bool { + return flow.GetSourcePrincipal() == nil && flow.GetDestPrincipal() == nil && + flow.GetHttpMethod() == nil && flow.GetHttpPath() == nil && + len(flow.GetSourceLabels()) == 0 && len(flow.GetDestLabels()) == 0 +} + +// endpointMatchesSourcePorts reports whether any rule that can apply to the endpoint in the scope +// and direction matches on the source port. A policy or profile the store does not hold counts as +// if it did: the evaluation will fail or change once it arrives, and until then the finer key +// only costs hit rate. +func endpointMatchesSourcePorts(store *policystore.PolicyStore, ep *proto.WorkloadEndpoint, scope PolicyScope, dir rules.RuleDir) bool { + for _, tier := range ep.Tiers { + for _, pID := range getPoliciesByDirection(dir, tier) { + if scope == EnforcedOnly && model.KindIsStaged(pID.Kind) { + continue + } + policy := store.PolicyByID[ftypes.ProtoToPolicyID(pID)] + if policy == nil { + return true + } + if dir == rules.RuleDirEgress && rulesMatchSourcePorts(policy.OutboundRules) || + dir != rules.RuleDirEgress && rulesMatchSourcePorts(policy.InboundRules) { + return true + } + } + } + for _, name := range ep.ProfileIds { + profile := store.ProfileByID[ftypes.ProtoToProfileID(&proto.ProfileID{Name: name})] + if profile == nil { + return true + } + if dir == rules.RuleDirEgress && rulesMatchSourcePorts(profile.OutboundRules) || + dir != rules.RuleDirEgress && rulesMatchSourcePorts(profile.InboundRules) { + return true + } + } + return false +} + +func rulesMatchSourcePorts(rs []*proto.Rule) bool { + for _, r := range rs { + if len(r.SrcPorts) > 0 || len(r.NotSrcPorts) > 0 || + len(r.SrcNamedPortIpSetIds) > 0 || len(r.NotSrcNamedPortIpSetIds) > 0 { + return true + } + } + return false +} diff --git a/app-policy/checker/verdictcache_test.go b/app-policy/checker/verdictcache_test.go new file mode 100644 index 00000000000..28cfb7a8bfc --- /dev/null +++ b/app-policy/checker/verdictcache_test.go @@ -0,0 +1,259 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package checker + +import ( + "fmt" + "net" + "testing" + + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + log "github.com/sirupsen/logrus" + + "github.com/projectcalico/calico/app-policy/policyscale" + "github.com/projectcalico/calico/app-policy/policystore" + "github.com/projectcalico/calico/felix/calc" + "github.com/projectcalico/calico/felix/proto" + "github.com/projectcalico/calico/felix/rules" + "github.com/projectcalico/calico/felix/types" +) + +// TestCachedEvaluateAgreesWithUncached runs the differential corpus, plus a repeat-heavy sample so +// that hits are exercised as well as misses, through a cached store and an uncached one, twice, +// so the second pass is answered from the cache. +func TestCachedEvaluateAgreesWithUncached(t *testing.T) { + _, restoreLogging := withBenchLogging(log.ErrorLevel) + defer restoreLogging() + + for _, c := range []struct { + name string + spec policyscale.Spec + }{ + {"egress allow-list", policyscale.EgressAllowList()}, + {"composite", policyscale.Composite()}, + } { + t.Run(c.name, func(t *testing.T) { + fx := policyscale.Build(c.spec) + plain, ep := fx.NewStore(), fx.Endpoint() + stats := &policystore.VerdictCacheStats{} + cached := fx.NewStore() + cached.Verdicts = policystore.NewVerdictCache(1<<16, stats) + + corpus := differentialCorpus(fx, 13, 150, 401) + for _, dir := range []policyscale.Direction{policyscale.Ingress, policyscale.Egress} { + if fx.Rules(dir) == 0 { + continue + } + s := fx.NewSampler(17, policyscale.FlowModel{Direction: dir, MissFraction: 0.1, RepeatFraction: 0.8}) + for i := 0; i < 300; i++ { + corpus = append(corpus, corpusFlow{name: fmt.Sprintf("%s repeat-heavy %d", dir, i), dir: ruleDir(dir), flow: s.Next()}) + } + } + + viaCache := func(scope PolicyScope, dir rules.RuleDir, _ *policystore.PolicyStore, ep *proto.WorkloadEndpoint, flow Flow) ([]*calc.RuleID, error) { + return Evaluate(scope, dir, cached, ep, flow) + } + for pass := 0; pass < 2; pass++ { + for _, scope := range []PolicyScope{StagedAsEnforced, EnforcedOnly} { + assertEquivalent(t, fmt.Sprintf("%s pass %d scope %d", c.name, pass, scope), scope, viaCache, Evaluate, plain, ep, corpus) + } + } + // The second pass repeats every flow of the first, so at least that many hits. + if hits, misses := stats.Hits.Load(), stats.Misses.Load(); hits < uint64(2*len(corpus)) || misses == 0 { + t.Errorf("cache traffic: %d hits, %d misses over %d flows x 2 scopes x 2 passes", hits, misses, len(corpus)) + } + if stats.Resets.Load() != 0 || stats.Evictions.Load() != 0 { + t.Errorf("unexpected resets %d / evictions %d", stats.Resets.Load(), stats.Evictions.Load()) + } + }) + } +} + +func TestCachedEvaluateNamedCases(t *testing.T) { + _, restoreLogging := withBenchLogging(log.ErrorLevel) + defer restoreLogging() + runNamedCases(t, cachedEvaluator(t)) +} + +// cachedEvaluator attaches a cache to the case's store, evaluates twice so that the second answer +// is served from the cache, checks the two agree, and returns the second. +func cachedEvaluator(t *testing.T) evaluator { + return func(scope PolicyScope, dir rules.RuleDir, store *policystore.PolicyStore, ep *proto.WorkloadEndpoint, flow Flow) ([]*calc.RuleID, error) { + if store.Verdicts == nil { + store.Verdicts = policystore.NewVerdictCache(1024, nil) + } + first, err1 := Evaluate(scope, dir, store, ep, flow) + second, err2 := Evaluate(scope, dir, store, ep, flow) + if (err1 != nil) != (err2 != nil) || !sameTrace(first, second) { + t.Errorf("cached evaluation differs from the first: %v (%v) vs %v (%v)", formatTrace(first), err1, formatTrace(second), err2) + } + return second, err2 + } +} + +// TestVerdictCacheFollowsStoreUpdates feeds a store through ProcessUpdate, as the collector does, +// and checks that a cached verdict is dropped as soon as anything the verdict depends on changes. +func TestVerdictCacheFollowsStoreUpdates(t *testing.T) { + _, restoreLogging := withBenchLogging(log.ErrorLevel) + defer restoreLogging() + + stats := &policystore.VerdictCacheStats{} + store := policystore.NewPolicyStore() + store.Verdicts = policystore.NewVerdictCache(1024, stats) + apply := func(u *proto.ToDataplane) { store.ProcessUpdate("per-host-policies", u) } + + pID := &proto.PolicyID{Name: "p", Kind: v3.KindGlobalNetworkPolicy} + apply(&proto.ToDataplane{Payload: &proto.ToDataplane_IpsetUpdate{IpsetUpdate: &proto.IPSetUpdate{Id: "s", Type: proto.IPSetUpdate_NET}}}) + apply(&proto.ToDataplane{Payload: &proto.ToDataplane_ActivePolicyUpdate{ActivePolicyUpdate: &proto.ActivePolicyUpdate{ + Id: pID, Policy: &proto.Policy{Tier: "t", InboundRules: []*proto.Rule{{Action: "allow", DstIpSetIds: []string{"s"}}}}, + }}}) + ep := &proto.WorkloadEndpoint{Tiers: []*proto.TierInfo{{Name: "t", DefaultAction: "Deny", IngressPolicies: []*proto.PolicyID{pID}}}} + flow := policyscale.NewFlow("10.0.0.9", 1234, "10.0.0.5", 80) + + eval := func() []*calc.RuleID { + t.Helper() + trace, err := Evaluate(StagedAsEnforced, rules.RuleDirIngress, store, ep, flow) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return trace + } + deny := calc.NewRuleID(v3.KindGlobalNetworkPolicy, "t", "p", "", tierDefaultActionIndex, rules.RuleDirIngress, rules.RuleActionDeny) + allow := calc.NewRuleID(v3.KindGlobalNetworkPolicy, "t", "p", "", 0, rules.RuleDirIngress, rules.RuleActionAllow) + + if got := eval(); !sameTrace(got, []*calc.RuleID{deny}) { + t.Fatalf("before the set has the address: %v", formatTrace(got)) + } + eval() + if stats.Hits.Load() != 1 { + t.Fatalf("second evaluation was not a hit: %+v", stats) + } + + // The set gains the destination: the verdict flips and the stale entry is gone. + apply(&proto.ToDataplane{Payload: &proto.ToDataplane_IpsetDeltaUpdate{IpsetDeltaUpdate: &proto.IPSetDeltaUpdate{Id: "s", AddedMembers: []string{"10.0.0.5/32"}}}}) + if got := eval(); !sameTrace(got, []*calc.RuleID{allow}) { + t.Fatalf("after the set gained the address: %v", formatTrace(got)) + } + if stats.Resets.Load() != 1 { + t.Fatalf("expected one reset, stats %+v", stats) + } + + // The policy changes its action: same again. + apply(&proto.ToDataplane{Payload: &proto.ToDataplane_ActivePolicyUpdate{ActivePolicyUpdate: &proto.ActivePolicyUpdate{ + Id: pID, Policy: &proto.Policy{Tier: "t", InboundRules: []*proto.Rule{{Action: "deny", DstIpSetIds: []string{"s"}}}}, + }}}) + denyRule := calc.NewRuleID(v3.KindGlobalNetworkPolicy, "t", "p", "", 0, rules.RuleDirIngress, rules.RuleActionDeny) + if got := eval(); !sameTrace(got, []*calc.RuleID{denyRule}) { + t.Fatalf("after the policy changed: %v", formatTrace(got)) + } + + // The policy goes away: the evaluation fails, and the failure is not cached. + apply(&proto.ToDataplane{Payload: &proto.ToDataplane_ActivePolicyRemove{ActivePolicyRemove: &proto.ActivePolicyRemove{Id: pID}}}) + for i := 0; i < 2; i++ { + if _, err := Evaluate(StagedAsEnforced, rules.RuleDirIngress, store, ep, flow); err == nil { + t.Fatal("expected the evaluation to fail with the policy missing") + } + } + if store.Verdicts.Len() != 0 { + t.Fatalf("a failed evaluation was cached: %d entries", store.Verdicts.Len()) + } +} + +// TestVerdictKeyIncludesSourcePortOnlyWhenRulesUseIt: flows that differ only in their source port +// share an entry unless a rule that applies to the endpoint looks at source ports. +func TestVerdictKeyIncludesSourcePortOnlyWhenRulesUseIt(t *testing.T) { + build := func(rule *proto.Rule) (*policystore.PolicyStore, *proto.WorkloadEndpoint, *policystore.VerdictCacheStats) { + stats := &policystore.VerdictCacheStats{} + store := policystore.NewPolicyStore() + store.Verdicts = policystore.NewVerdictCache(1024, stats) + pID := &proto.PolicyID{Name: "p", Kind: v3.KindGlobalNetworkPolicy} + store.PolicyByID[types.ProtoToPolicyID(pID)] = &proto.Policy{Tier: "t", InboundRules: []*proto.Rule{rule}} + ep := &proto.WorkloadEndpoint{Tiers: []*proto.TierInfo{{Name: "t", DefaultAction: "Deny", IngressPolicies: []*proto.PolicyID{pID}}}} + return store, ep, stats + } + allowed := func(trace []*calc.RuleID) bool { return len(trace) == 1 && trace[0].Action == rules.RuleActionAllow } + + // Rules ignore the source port: one entry serves every source port. + store, ep, stats := build(&proto.Rule{Action: "allow", DstPorts: []*proto.PortRange{{First: 80, Last: 80}}}) + for sp := 1000; sp < 1010; sp++ { + trace, err := Evaluate(StagedAsEnforced, rules.RuleDirIngress, store, ep, policyscale.NewFlow("10.0.0.9", sp, "10.0.0.5", 80)) + if err != nil || !allowed(trace) { + t.Fatalf("source port %d: %v %v", sp, formatTrace(trace), err) + } + } + if stats.Misses.Load() != 1 || stats.Hits.Load() != 9 { + t.Fatalf("source-port-insensitive endpoint: %d misses, %d hits, want 1 and 9", stats.Misses.Load(), stats.Hits.Load()) + } + + // A rule matches on the source port: the key carries it, and verdicts stay per port. + store, ep, stats = build(&proto.Rule{Action: "allow", SrcPorts: []*proto.PortRange{{First: 1000, Last: 1004}}}) + for pass := 0; pass < 2; pass++ { + for sp := 1000; sp < 1010; sp++ { + trace, err := Evaluate(StagedAsEnforced, rules.RuleDirIngress, store, ep, policyscale.NewFlow("10.0.0.9", sp, "10.0.0.5", 80)) + if err != nil || allowed(trace) != (sp <= 1004) { + t.Fatalf("pass %d source port %d: %v %v", pass, sp, formatTrace(trace), err) + } + } + } + if stats.Misses.Load() != 10 || stats.Hits.Load() != 10 { + t.Fatalf("source-port-sensitive endpoint: %d misses, %d hits, want 10 and 10", stats.Misses.Load(), stats.Hits.Load()) + } + + // A negated named source port counts as looking at the source port too. + store, ep, stats = build(&proto.Rule{Action: "allow", NotSrcNamedPortIpSetIds: []string{"np"}}) + for sp := 1000; sp < 1003; sp++ { + _, _ = Evaluate(StagedAsEnforced, rules.RuleDirIngress, store, ep, policyscale.NewFlow("10.0.0.9", sp, "10.0.0.5", 80)) + } + if stats.Misses.Load() != 3 { + t.Fatalf("named source port endpoint: %d misses, want 3", stats.Misses.Load()) + } +} + +func TestVerdictCacheSkipsL7FlowsAndNilAddresses(t *testing.T) { + stats := &policystore.VerdictCacheStats{} + store := policystore.NewPolicyStore() + store.Verdicts = policystore.NewVerdictCache(1024, stats) + pID := &proto.PolicyID{Name: "p", Kind: v3.KindGlobalNetworkPolicy} + store.PolicyByID[types.ProtoToPolicyID(pID)] = &proto.Policy{Tier: "t", InboundRules: []*proto.Rule{{Action: "allow"}}} + ep := &proto.WorkloadEndpoint{Tiers: []*proto.TierInfo{{Name: "t", DefaultAction: "Deny", IngressPolicies: []*proto.PolicyID{pID}}}} + + principal := "spiffe://cluster.local/ns/default/sa/client" + method := "GET" + flows := map[string]Flow{ + "source principal": &MockFlow{SourceIP: net.ParseIP("10.0.0.9"), DestIP: net.ParseIP("10.0.0.5"), DestPort: 80, Protocol: 6, SourcePrincipal: &principal}, + "HTTP method": &MockFlow{SourceIP: net.ParseIP("10.0.0.9"), DestIP: net.ParseIP("10.0.0.5"), DestPort: 80, Protocol: 6, HttpMethod: &method}, + "destination labels": &MockFlow{SourceIP: net.ParseIP("10.0.0.9"), DestIP: net.ParseIP("10.0.0.5"), DestPort: 80, Protocol: 6, DestLabels: map[string]string{"a": "b"}}, + "nil source": &policyscale.Flow{DstIP: net.ParseIP("10.0.0.5"), DstPort: 80, Protocol: 6}, + "nil destination": &policyscale.Flow{SrcIP: net.ParseIP("10.0.0.9"), DstPort: 80, Protocol: 6}, + } + for name, f := range flows { + for i := 0; i < 2; i++ { + if _, err := Evaluate(StagedAsEnforced, rules.RuleDirIngress, store, ep, f); err != nil { + t.Fatalf("%s: %v", name, err) + } + } + } + if stats.Hits.Load()+stats.Misses.Load() != 0 || store.Verdicts.Len() != 0 { + t.Fatalf("uncacheable flows touched the cache: %+v, %d entries", stats, store.Verdicts.Len()) + } + + // And a plain L4 flow on the same store is cached. + for i := 0; i < 2; i++ { + _, _ = Evaluate(StagedAsEnforced, rules.RuleDirIngress, store, ep, policyscale.NewFlow("10.0.0.9", 1, "10.0.0.5", 80)) + } + if stats.Hits.Load() != 1 || stats.Misses.Load() != 1 { + t.Fatalf("L4 flow: %+v", stats) + } +} diff --git a/app-policy/policyscale/build.go b/app-policy/policyscale/build.go new file mode 100644 index 00000000000..c1d2f09eed1 --- /dev/null +++ b/app-policy/policyscale/build.go @@ -0,0 +1,442 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policyscale + +import ( + "fmt" + "math/rand" + "net/netip" + + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + + "github.com/projectcalico/calico/felix/proto" +) + +const ( + // SentinelIPSetID names the set that holds the denied flow's addresses. Baseline rules that + // reference an IP set also carry a negated reference to it; see BaselineParams. + SentinelIPSetID = "policyscale-sentinel" + + baselineIPSetPrefix = "policyscale-base-" + egressIPSetPrefix = "policyscale-egress-" + + // BaselineGuardPort is the destination port that guards baseline rules with no IP set + // reference. No generated flow uses it, so those rules never match. + BaselineGuardPort int32 = 65001 + + // EgressTailPort is outside the range egress rules draw their ports from, so only the target + // rule carries it: the extreme of the measured tail, where 90% of ports are on 5 rules or + // fewer. + EgressTailPort int32 = 60999 + + // egressTailPortMin/Range bound the long tail of otherwise-unique destination ports. + egressTailPortMin = 1024 + egressTailPortRange = 55000 +) + +// Fixture is a built policy set: the model the oracle and flow generators reason about, plus the +// proto policies the store and endpoint are made from. +type Fixture struct { + Spec Spec + + tiers []*tierModel + sets map[string]*ipSetModel + setOrder []string + + missingRefs int + egressTarget *EgressTarget +} + +// EgressTarget describes the one egress rule a matching flow is built to hit when +// EgressParams.TargetDepth is set. +type EgressTarget struct { + Policy string // Policy name, as the trace reports it. + RuleIndex int // Index of the rule within its policy. + Ordinal int // Position of the rule in the egress walk, from 0. + RulesWalked int // Rules visited before and including it. + AddrInCIDR string // An address inside the rule's destination CIDR. + TailPort int32 // A port only the target carries. + PopularPort int32 // The most popular port, which the target also carries. +} + +type tierModel struct { + name string + defaultAction string // "Deny" or "Pass", as TierInfo.DefaultAction spells it. + policies [2][]*policyModel +} + +type policyModel struct { + name string + rules []*ruleModel + + id *proto.PolicyID + policy *proto.Policy +} + +// ruleModel is what the generator knows about a rule. Every criterion is ANDed; an unset one is +// unconstrained. The oracle evaluates this, never the proto. +type ruleModel struct { + action string // "allow", "deny" or "pass". + dstPorts []int32 + dstNet netip.Prefix + srcSet, notSrcSet, dstSet, notDstSet string +} + +type ipSetModel struct { + id string + members []string // CIDR strings, as the store and the resources take them. + addrs map[netip.Addr]struct{} + missing bool +} + +// Build materialises a Spec. Each preset draws from its own generator seeded with Spec.Seed, so a +// preset builds the same rules whether it is used alone or in a composite. +func Build(spec Spec) *Fixture { + fx := &Fixture{Spec: spec, sets: map[string]*ipSetModel{}} + if spec.Baseline != nil { + fx.buildBaseline(rand.New(rand.NewSource(spec.Seed)), *spec.Baseline) + } + if spec.Egress != nil { + fx.buildEgress(rand.New(rand.NewSource(spec.Seed)), *spec.Egress) + } + for _, t := range fx.tiers { + for dir, policies := range t.policies { + for _, p := range policies { + p.buildProto(t.name, Direction(dir)) + } + } + } + return fx +} + +func (fx *Fixture) buildBaseline(rng *rand.Rand, p BaselineParams) { + setIDs := fx.makeBaselineIPSets(rng, p.SizeHistogram) + + // Pick the rule slots that get the few non-Pass actions. + numRules := p.Policies * p.RulesPerPolicy + specialAction := map[int]string{} + for len(specialAction) < p.DenyRules { + specialAction[rng.Intn(numRules)] = "deny" + } + for n := 0; n < p.AllowRules; { + slot := rng.Intn(numRules) + if _, ok := specialAction[slot]; !ok { + specialAction[slot] = "allow" + n++ + } + } + + tier := fx.tier(p.Tier, "Deny") + var referencedIDs []string + refCount := map[string]int{} + ruleIdx := 0 + for i := 0; i < p.Policies; i++ { + pol := &policyModel{name: fmt.Sprintf("policy-%03d", i)} + for j := 0; j < p.RulesPerPolicy; j++ { + action := specialAction[ruleIdx] + if action == "" { + action = "pass" + } + r := &ruleModel{action: action} + var refID string + if rng.Float64() < p.IPSetRefFraction { + refID = setIDs[rng.Intn(len(setIDs))] + if rng.Intn(2) == 0 { + r.srcSet, r.notSrcSet = refID, SentinelIPSetID + } else { + r.dstSet, r.notDstSet = refID, SentinelIPSetID + } + } else { + r.dstPorts = []int32{BaselineGuardPort} + } + pol.rules = append(pol.rules, r) + if refID != "" { + if refCount[refID] == 0 { + referencedIDs = append(referencedIDs, refID) + } + refCount[refID]++ + } + ruleIdx++ + } + tier.policies[Ingress] = append(tier.policies[Ingress], pol) + } + + // Leave some referenced sets out of the store to reproduce the "IPSet not found" storm. + rng.Shuffle(len(referencedIDs), func(a, b int) { + referencedIDs[a], referencedIDs[b] = referencedIDs[b], referencedIDs[a] + }) + for _, id := range referencedIDs[:min(p.MissingIPSets, len(referencedIDs))] { + fx.sets[id].missing = true + fx.missingRefs += refCount[id] + } +} + +// makeBaselineIPSets adds the sentinel set and the NET sets the baseline rules reference, +// following the size histogram. Members are unique /32s from 10.0.0.0/9. +func (fx *Fixture) makeBaselineIPSets(rng *rand.Rand, hist []SizeBucket) []string { + fx.addSet(SentinelIPSetID, []string{SourceIP + "/32", DeniedDestIP + "/32"}) + + var ids []string + member := 0 + for _, bucket := range hist { + for i := 0; i < bucket.Sets; i++ { + id := fmt.Sprintf("%s%04d", baselineIPSetPrefix, len(ids)) + size := bucket.MinSize + rng.Intn(bucket.MaxSize-bucket.MinSize+1) + members := make([]string, 0, size) + for j := 0; j < size; j++ { + members = append(members, fmt.Sprintf("10.%d.%d.%d/32", member>>16&0x7f, member>>8&0xff, member&0xff)) + member++ + } + fx.addSet(id, members) + ids = append(ids, id) + } + } + return ids +} + +func (fx *Fixture) buildEgress(rng *rand.Rand, p EgressParams) { + setIDs := fx.makeEgressIPSets(p.IPSets, p.MembersPerSet) + + numRules := p.Policies * p.RulesPerPolicy + targetRule := -1 + if p.TargetDepth > 0 { + targetRule = int(float64(numRules) * p.TargetDepth) + } + + tier := fx.tier(p.Tier, "Deny") + nextCIDR := 0 + ruleIdx := 0 + for i := 0; i < p.Policies; i++ { + pol := &policyModel{name: fmt.Sprintf("egress-%03d", i)} + for j := 0; j < p.RulesPerPolicy; j++ { + // All rules are Pass bar the target: a rule's action is only consulted once it + // matches, so the action mix does not affect the walk. + r := &ruleModel{action: "pass", dstPorts: makeEgressRulePorts(rng, p)} + if rng.Float64() < p.IPSetRuleFraction { + r.dstSet = setIDs[rng.Intn(len(setIDs))] + } else { + r.dstNet = egressCIDR(nextCIDR) + nextCIDR++ + } + if ruleIdx == targetRule { + // Make the target reachable on address and on a port of its own, and give it a + // distinct action so a trace assertion is unambiguous. It carries both the tail + // port and the popular port, so flows aimed at it differ only in how many rules + // along the way survive the port comparison. + r.action = "allow" + r.dstNet = egressCIDR(nextCIDR) + r.dstSet = "" + r.dstPorts = append(r.dstPorts, EgressTailPort, p.PortWeights[0].Port) + fx.egressTarget = &EgressTarget{ + Policy: pol.name, + RuleIndex: j, + Ordinal: ruleIdx, + RulesWalked: ruleIdx + 1, + AddrInCIDR: egressAddrInCIDR(nextCIDR), + TailPort: EgressTailPort, + PopularPort: p.PortWeights[0].Port, + } + nextCIDR++ + } + pol.rules = append(pol.rules, r) + ruleIdx++ + } + tier.policies[Egress] = append(tier.policies[Egress], pol) + } +} + +// makeEgressIPSets adds the NET sets the selector-matching egress rules reference. Members are +// /32s from 10.128.0.0/9, kept clear of the rule CIDRs handed out from 10.0.0.0/9. +func (fx *Fixture) makeEgressIPSets(numSets, membersPerSet int) []string { + ids := make([]string, 0, numSets) + member := 0 + for i := 0; i < numSets; i++ { + id := fmt.Sprintf("%s%04d", egressIPSetPrefix, i) + members := make([]string, 0, membersPerSet) + for j := 0; j < membersPerSet; j++ { + members = append(members, fmt.Sprintf("10.%d.%d.%d/32", 128+(member>>16&0x7f), member>>8&0xff, member&0xff)) + member++ + } + fx.addSet(id, members) + ids = append(ids, id) + } + return ids +} + +// makeEgressRulePorts picks a rule's destination ports: the measured head with its measured +// frequency, topped up from a wide tail. +func makeEgressRulePorts(rng *rand.Rand, p EgressParams) []int32 { + var ports []int32 + for _, w := range p.PortWeights { + if rng.Float64() < w.Fraction { + ports = append(ports, w.Port) + } + } + for _, n := range p.PortsPerRule { + if rng.Float64() < n.Fraction { + for len(ports) < n.Count { + ports = append(ports, int32(egressTailPortMin+rng.Intn(egressTailPortRange))) + } + break + } + } + if len(ports) == 0 { + ports = append(ports, int32(egressTailPortMin+rng.Intn(egressTailPortRange))) + } + return ports +} + +func egressCIDR(n int) netip.Prefix { + return netip.MustParsePrefix(fmt.Sprintf("10.%d.%d.0/24", n>>8&0x7f, n&0xff)) +} + +func egressAddrInCIDR(n int) string { + return fmt.Sprintf("10.%d.%d.7", n>>8&0x7f, n&0xff) +} + +func (fx *Fixture) tier(name, defaultAction string) *tierModel { + for _, t := range fx.tiers { + if t.name == name { + return t + } + } + t := &tierModel{name: name, defaultAction: defaultAction} + fx.tiers = append(fx.tiers, t) + return t +} + +func (fx *Fixture) addSet(id string, members []string) { + if _, ok := fx.sets[id]; ok { + return + } + s := &ipSetModel{id: id, members: members, addrs: make(map[netip.Addr]struct{}, len(members))} + for _, m := range members { + s.addrs[netip.MustParsePrefix(m).Addr().Unmap()] = struct{}{} + } + fx.sets[id] = s + fx.setOrder = append(fx.setOrder, id) +} + +func (p *policyModel) buildProto(tier string, dir Direction) { + protoRules := make([]*proto.Rule, len(p.rules)) + for i, r := range p.rules { + protoRules[i] = r.proto() + } + p.id = &proto.PolicyID{Name: p.name, Kind: v3.KindGlobalNetworkPolicy} + p.policy = &proto.Policy{Tier: tier} + if dir == Egress { + p.policy.OutboundRules = protoRules + } else { + p.policy.InboundRules = protoRules + } +} + +func (r *ruleModel) proto() *proto.Rule { + pr := &proto.Rule{Action: r.action} + for _, port := range r.dstPorts { + pr.DstPorts = append(pr.DstPorts, &proto.PortRange{First: port, Last: port}) + } + if r.dstNet.IsValid() { + pr.DstNet = []string{r.dstNet.String()} + } + if r.srcSet != "" { + pr.SrcIpSetIds = []string{r.srcSet} + } + if r.notSrcSet != "" { + pr.NotSrcIpSetIds = []string{r.notSrcSet} + } + if r.dstSet != "" { + pr.DstIpSetIds = []string{r.dstSet} + } + if r.notDstSet != "" { + pr.NotDstIpSetIds = []string{r.notDstSet} + } + return pr +} + +// Tiers returns the tier names in evaluation order. +func (fx *Fixture) Tiers() []string { + names := make([]string, len(fx.tiers)) + for i, t := range fx.tiers { + names[i] = t.name + } + return names +} + +// Policies returns the number of policies with rules in the given direction. +func (fx *Fixture) Policies(dir Direction) int { + n := 0 + for _, t := range fx.tiers { + n += len(t.policies[dir]) + } + return n +} + +// Rules returns the number of rules a flow in the given direction walks when nothing matches. +func (fx *Fixture) Rules(dir Direction) int { + n := 0 + for _, t := range fx.tiers { + for _, p := range t.policies[dir] { + n += len(p.rules) + } + } + return n +} + +// IPSets returns the number of IP sets the policies reference, including any left out of the +// store; MissingIPSets says how many of those are left out. +func (fx *Fixture) IPSets() int { return len(fx.setOrder) } + +func (fx *Fixture) MissingIPSets() int { + n := 0 + for _, id := range fx.setOrder { + if fx.sets[id].missing { + n++ + } + } + return n +} + +// IPSetMembers returns the total number of members across the IP sets present in the store. +func (fx *Fixture) IPSetMembers() int { + n := 0 + for _, id := range fx.setOrder { + if s := fx.sets[id]; !s.missing { + n += len(s.members) + } + } + return n +} + +// MissingSetReferences returns the number of rule references to sets left out of the store, +// which is exactly the number of "IPSet not found" warnings one evaluation of a flow that walks +// the whole baseline set emits. +func (fx *Fixture) MissingSetReferences() int { return fx.missingRefs } + +// EgressTarget returns the target rule, or nil when the egress preset has none. +func (fx *Fixture) EgressTarget() *EgressTarget { return fx.egressTarget } + +// ruleAt returns the policy and rule at the given position of the walk in a direction. +func (fx *Fixture) ruleAt(dir Direction, ordinal int) (*policyModel, *ruleModel, bool) { + for _, t := range fx.tiers { + for _, p := range t.policies[dir] { + if ordinal < len(p.rules) { + return p, p.rules[ordinal], true + } + ordinal -= len(p.rules) + } + } + return nil, nil, false +} diff --git a/app-policy/policyscale/doc.go b/app-policy/policyscale/doc.go new file mode 100644 index 00000000000..87179728b66 --- /dev/null +++ b/app-policy/policyscale/doc.go @@ -0,0 +1,42 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package policyscale generates the large synthetic policy sets used to benchmark and test the +// user-mode policy engine in app-policy/checker and its caller in Felix's flow-log collector. +// +// One generator, two outputs. A Fixture built from a Spec can be materialised as a +// policystore.PolicyStore plus a proto.WorkloadEndpoint (what the engine and the collector +// evaluate against, in unit tests and Go benchmarks) or as Calico resources (Tier, +// GlobalNetworkPolicy, GlobalNetworkSet) that apply the same policy set to a real cluster. Both +// come from the same model, so a number measured at the engine level, at the collector level and +// on a node all describe the same policy set. +// +// The presets are the two rule-set shapes measured in production diagnostics from a large +// deployment, anonymised: +// +// - Baseline: hundreds of "baseline" policies that apply to every endpoint, almost every rule a +// Pass, about a quarter referencing one of thousands of IP sets whose sizes follow the +// measured histogram. See DefaultBaseline. +// - EgressAllowList: a single-tier destination allow-list of ~300 policies and ~18.6k egress +// rules, each matching a destination address (a CIDR for ~75%, a selector-derived IP set for +// ~25%) and a handful of destination ports drawn from the measured port distribution. See +// DefaultEgress. +// - Composite: both applied to one endpoint, the reference set the PMREQ-954 targets are quoted +// against. +// +// The fixture also knows the answer. Fixture.Expect computes the verdict the engine must reach +// for a flow from the generator's own model of each rule, independently of the engine's matching +// code, so it serves as the oracle for differential tests; MatchingFlow, DeniedFlow and Sampler +// produce flows that exercise the walk at chosen depths or following a configurable flow model. +package policyscale diff --git a/app-policy/policyscale/flow.go b/app-policy/policyscale/flow.go new file mode 100644 index 00000000000..369cf09b888 --- /dev/null +++ b/app-policy/policyscale/flow.go @@ -0,0 +1,182 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policyscale + +import ( + "fmt" + "math/rand" + "net" +) + +// The addresses and ports the denied flows use. TEST-NET addresses: generated IP set members and +// rule CIDRs all come from 10.0.0.0/8, so no rule matches them on address and the walk covers the +// whole policy set. +const ( + SourceIP = "192.0.2.10" + DeniedDestIP = "198.51.100.20" + // UnlistedIP is in no generated set and no rule CIDR, and unlike the two above it is not in + // the sentinel set either. + UnlistedIP = "203.0.113.1" + + DefaultSourcePort = 45000 + // BaselineDestPort is the denied ingress flow's destination port; no baseline rule uses it. + BaselineDestPort = 8080 + + ProtocolTCP = 6 +) + +// Flow is an L4 flow with no L7 attributes, as the collector presents flows to the engine. It +// satisfies checker.Flow. +type Flow struct { + SrcIP, DstIP net.IP + SrcPort, DstPort int + Protocol int +} + +func (f *Flow) GetSourceIP() net.IP { return f.SrcIP } +func (f *Flow) GetDestIP() net.IP { return f.DstIP } +func (f *Flow) GetSourcePort() int { return f.SrcPort } +func (f *Flow) GetDestPort() int { return f.DstPort } +func (f *Flow) GetProtocol() int { return f.Protocol } +func (f *Flow) GetHttpMethod() *string { return nil } +func (f *Flow) GetHttpPath() *string { return nil } +func (f *Flow) GetSourcePrincipal() *string { return nil } +func (f *Flow) GetDestPrincipal() *string { return nil } +func (f *Flow) GetSourceLabels() map[string]string { return nil } +func (f *Flow) GetDestLabels() map[string]string { return nil } + +func (f *Flow) String() string { + return fmt.Sprintf("%d %s:%d->%s:%d", f.Protocol, f.SrcIP, f.SrcPort, f.DstIP, f.DstPort) +} + +// NewFlow builds a TCP flow. +func NewFlow(srcIP string, srcPort int, dstIP string, dstPort int) *Flow { + return &Flow{ + SrcIP: net.ParseIP(srcIP), + DstIP: net.ParseIP(dstIP), + SrcPort: srcPort, + DstPort: dstPort, + Protocol: ProtocolTCP, + } +} + +// DeniedFlow returns a flow that no rule in the given direction matches, so an evaluation walks +// the whole policy set and ends in the tier default deny. In the egress direction it uses the most +// popular port, so that the address criteria of every rule sharing that port are evaluated too. +func (fx *Fixture) DeniedFlow(dir Direction) *Flow { + port := BaselineDestPort + if dir == Egress && fx.Spec.Egress != nil && len(fx.Spec.Egress.PortWeights) > 0 { + port = int(fx.Spec.Egress.PortWeights[0].Port) + } + return NewFlow(SourceIP, DefaultSourcePort, DeniedDestIP, port) +} + +// MatchingFlow returns a flow aimed at the rule at the given position of the walk in a direction: +// one that satisfies that rule's criteria. An earlier rule with overlapping criteria may match it +// first; Expect gives the actual verdict. A flow aimed at a rule that cannot match (an empty IP +// set, or a rule whose position is out of range) is a denied flow. +func (fx *Fixture) MatchingFlow(dir Direction, ordinal int) *Flow { + f := fx.DeniedFlow(dir) + _, r, ok := fx.ruleAt(dir, ordinal) + if !ok { + return f + } + if len(r.dstPorts) > 0 { + f.DstPort = int(r.dstPorts[0]) + } + switch { + case r.dstNet.IsValid(): + f.DstIP = net.ParseIP(r.dstNet.Addr().Next().Next().String()) + case r.dstSet != "": + f.DstIP = fx.addrMatching(r.dstSet) + case r.srcSet != "": + f.SrcIP = fx.addrMatching(r.srcSet) + } + return f +} + +// addrMatching returns an address that satisfies a positive reference to the set: its first +// member, or, for a set the store does not hold (the engine skips those), an address outside the +// sentinel set. Nil when the set is present but empty, which no address satisfies. +func (fx *Fixture) addrMatching(setID string) net.IP { + s := fx.sets[setID] + if s.missing { + return net.ParseIP(UnlistedIP) + } + if len(s.members) == 0 { + return nil + } + ip, _, err := net.ParseCIDR(s.members[0]) + if err != nil { + panic(err) + } + return ip +} + +// FlowModel describes the mix of flows a Sampler produces. +type FlowModel struct { + Direction Direction + // MissFraction is the fraction of new flows that match no rule and walk the whole set. + MissFraction float64 + // RepeatFraction is the fraction of flows that repeat an earlier flow's addresses and + // destination port with a new source port, the way clients reconnect. It decides whether a + // verdict cache keyed without the source port can help. + RepeatFraction float64 + // RecentWindow is how many distinct flows repeats are drawn from. Default 1024. + RecentWindow int +} + +// Sampler produces flows following a FlowModel: matching flows are aimed uniformly at every depth +// of the walk, so the verdict mix follows the rule mix. +type Sampler struct { + fx *Fixture + rng *rand.Rand + model FlowModel + recent []*Flow +} + +// NewSampler returns a deterministic sampler for the fixture. +func (fx *Fixture) NewSampler(seed int64, model FlowModel) *Sampler { + if model.RecentWindow <= 0 { + model.RecentWindow = 1024 + } + return &Sampler{fx: fx, rng: rand.New(rand.NewSource(seed)), model: model} +} + +// Next returns the next flow. The returned flow is the caller's to keep. +func (s *Sampler) Next() *Flow { + if len(s.recent) > 0 && s.rng.Float64() < s.model.RepeatFraction { + f := *s.recent[s.rng.Intn(len(s.recent))] + f.SrcPort = s.ephemeralPort() + return &f + } + var f *Flow + if rules := s.fx.Rules(s.model.Direction); rules == 0 || s.rng.Float64() < s.model.MissFraction { + f = s.fx.DeniedFlow(s.model.Direction) + } else { + f = s.fx.MatchingFlow(s.model.Direction, s.rng.Intn(rules)) + } + f.SrcPort = s.ephemeralPort() + if len(s.recent) < s.model.RecentWindow { + s.recent = append(s.recent, f) + } else { + s.recent[s.rng.Intn(len(s.recent))] = f + } + return f +} + +func (s *Sampler) ephemeralPort() int { + return 32768 + s.rng.Intn(28232) +} diff --git a/app-policy/policyscale/oracle.go b/app-policy/policyscale/oracle.go new file mode 100644 index 00000000000..eed096deca9 --- /dev/null +++ b/app-policy/policyscale/oracle.go @@ -0,0 +1,135 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policyscale + +import ( + "fmt" + "net" + "net/netip" + "slices" + "strings" + + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" +) + +// Verdict is one entry of the rule trace the engine is expected to report: a Pass on the way, or +// the final Allow or Deny. It carries the fields of calc.RuleID that identify a rule. +type Verdict struct { + Kind string // GlobalNetworkPolicy, or Profile for the end-of-profiles deny. + Tier string + Policy string + Index int // Rule index in the policy; TierDefaultIndex for a tier or profile default. + Action string // allow, deny or pass. +} + +const ( + // TierDefaultIndex is the index the engine reports for a tier's default action, and for the + // deny that ends an evaluation with no matching profile. + TierDefaultIndex = -1 + // ProfileName is the tier and policy name the engine reports for that deny. + ProfileName = "__PROFILE__" +) + +func (v Verdict) String() string { + return fmt.Sprintf("%s/%s/%s[%d]=%s", v.Kind, v.Tier, v.Policy, v.Index, v.Action) +} + +// Expect returns the trace the engine must report for the flow in the given direction, computed +// from the generator's own model of each rule rather than from the proto the engine reads, so that +// the two are independent. It reproduces the engine's tier walk: policies of a tier in order, +// first matching rule decides; Allow or Deny ends the evaluation; Pass moves to the next tier; a +// tier none of whose policies matched applies its default action; and with no profiles on the +// endpoint, an evaluation that gets past every tier is denied. +func (fx *Fixture) Expect(dir Direction, f *Flow) []Verdict { + var trace []Verdict + for _, t := range fx.tiers { + policies := t.policies[dir] + if len(policies) == 0 { + continue + } + matched := false + Tier: + for _, p := range policies { + for i, r := range p.rules { + if !fx.ruleMatches(r, f) { + continue + } + matched = true + trace = append(trace, Verdict{Kind: v3.KindGlobalNetworkPolicy, Tier: t.name, Policy: p.name, Index: i, Action: r.action}) + if r.action == "pass" { + break Tier + } + return trace + } + } + if !matched { + action := strings.ToLower(t.defaultAction) + trace = append(trace, Verdict{Kind: v3.KindGlobalNetworkPolicy, Tier: t.name, Policy: policies[0].name, Index: TierDefaultIndex, Action: action}) + if action != "pass" { + return trace + } + } + } + return append(trace, Verdict{Kind: v3.KindProfile, Tier: ProfileName, Policy: ProfileName, Index: TierDefaultIndex, Action: "deny"}) +} + +// ruleMatches evaluates a rule's criteria against the flow. The criteria are the ones the +// generator emits; each mirrors the engine's semantics for that field: +// - a protocol outside 1..255 matches no rule; +// - a port list matches when it lists the flow's destination port; +// - a CIDR matches when it contains the destination address; +// - a positive IP set reference requires the address to be a member; a negated one requires it +// not to be; a reference to a set the store does not hold is skipped, as the engine skips it; +// - an address that is not an IP (nil) is a member of no set and inside no CIDR. +func (fx *Fixture) ruleMatches(r *ruleModel, f *Flow) bool { + if f.Protocol < 1 || f.Protocol > 255 { + return false + } + if len(r.dstPorts) > 0 && !slices.Contains(r.dstPorts, int32(f.DstPort)) { + return false + } + if r.dstNet.IsValid() { + a, ok := toAddr(f.DstIP) + if !ok || !r.dstNet.Contains(a) { + return false + } + } + return fx.setAllows(r.srcSet, f.SrcIP, true) && + fx.setAllows(r.notSrcSet, f.SrcIP, false) && + fx.setAllows(r.dstSet, f.DstIP, true) && + fx.setAllows(r.notDstSet, f.DstIP, false) +} + +// setAllows reports whether a (possibly negated) reference to a set lets the address through. +func (fx *Fixture) setAllows(setID string, ip net.IP, positive bool) bool { + if setID == "" { + return true + } + s := fx.sets[setID] + if s.missing { + return true + } + a, ok := toAddr(ip) + if !ok { + return !positive + } + _, member := s.addrs[a] + return member == positive +} + +func toAddr(ip net.IP) (netip.Addr, bool) { + a, ok := netip.AddrFromSlice(ip) + return a.Unmap(), ok +} diff --git a/app-policy/policyscale/policyscale_test.go b/app-policy/policyscale/policyscale_test.go new file mode 100644 index 00000000000..38b7dcecee3 --- /dev/null +++ b/app-policy/policyscale/policyscale_test.go @@ -0,0 +1,339 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policyscale_test + +import ( + "bytes" + "fmt" + "strings" + "testing" + + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/yaml" + + . "github.com/projectcalico/calico/app-policy/policyscale" + validator "github.com/projectcalico/calico/libcalico-go/lib/validator/v3" +) + +func TestPresetScale(t *testing.T) { + cases := []struct { + name string + spec Spec + ingressRules, egressRules int + ingressPolicies, egressPol int + sets int + }{ + {"baseline", Baseline(), 294 * 68, 0, 294, 0, 3708 + 1}, + {"egress", EgressAllowList(), 0, 301 * 62, 0, 301, 3862}, + {"composite", Composite(), 294 * 68, 301 * 62, 294, 301, 3708 + 1 + 3862}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + fx := Build(c.spec) + if got := fx.Rules(Ingress); got != c.ingressRules { + t.Errorf("ingress rules: got %d, want %d", got, c.ingressRules) + } + if got := fx.Rules(Egress); got != c.egressRules { + t.Errorf("egress rules: got %d, want %d", got, c.egressRules) + } + if got := fx.Policies(Ingress); got != c.ingressPolicies { + t.Errorf("ingress policies: got %d, want %d", got, c.ingressPolicies) + } + if got := fx.Policies(Egress); got != c.egressPol { + t.Errorf("egress policies: got %d, want %d", got, c.egressPol) + } + if got := fx.IPSets(); got != c.sets { + t.Errorf("IP sets: got %d, want %d", got, c.sets) + } + if got := fx.Tiers(); len(got) != 1 || got[0] != "perimeter" { + t.Errorf("tiers: got %v, want [perimeter]", got) + } + store := fx.NewStore() + if got := len(store.PolicyByID); got != c.ingressPolicies+c.egressPol { + t.Errorf("store policies: got %d", got) + } + if got := len(store.IPSetByID); got != c.sets { + t.Errorf("store IP sets: got %d, want %d", got, c.sets) + } + if got := len(fx.Updates()); got != c.sets+c.ingressPolicies+c.egressPol { + t.Errorf("updates: got %d", got) + } + }) + } +} + +func TestBaselineMembersMatchMeasurement(t *testing.T) { + fx := Build(Baseline()) + // The histogram sums to 3,708 sets. Sizes are drawn uniformly within each bucket, so the + // 1,000-9,999 bucket alone contributes ~870k members and the set holds about 1M in all, four + // times the measured ~256k; the measured distribution was skewed towards the small end of each + // bucket. Pin the generated total so that a change to it is deliberate. + if got := fx.IPSetMembers(); got < 950_000 || got > 1_100_000 { + t.Errorf("baseline members: got %d, want about 1M", got) + } +} + +func TestMissingSets(t *testing.T) { + spec := Baseline() + spec.Baseline.MissingIPSets = 8 + fx := Build(spec) + if got := fx.MissingIPSets(); got != 8 { + t.Fatalf("missing sets: got %d, want 8", got) + } + if fx.MissingSetReferences() < 8 { + t.Errorf("missing references: got %d, want at least one per missing set", fx.MissingSetReferences()) + } + if got := len(fx.NewStore().IPSetByID); got != 3709-8 { + t.Errorf("store sets: got %d, want %d", got, 3709-8) + } + // Every resource is still rendered: a cluster never has a missing set. + if got := countKind(fx.Resources(ResourceOptions{}), v3.KindGlobalNetworkSet); got != 3709 { + t.Errorf("rendered sets: got %d, want 3709", got) + } +} + +func TestDeterministic(t *testing.T) { + a, b := render(t, Build(Composite())), render(t, Build(Composite())) + if !bytes.Equal(a, b) { + t.Fatal("two builds of the same spec rendered differently") + } + other := Composite() + other.Seed++ + if bytes.Equal(a, render(t, Build(other))) { + t.Fatal("a different seed rendered the same resources") + } +} + +func TestEgressTargetAndOracle(t *testing.T) { + fx := Build(EgressAllowList()) + tgt := fx.EgressTarget() + if tgt == nil { + t.Fatal("no egress target") + } + numRules := 301 * 62 + if tgt.Ordinal != int(0.65*float64(numRules)) || tgt.RulesWalked != tgt.Ordinal+1 { + t.Errorf("target position: %+v", tgt) + } + want := []Verdict{{Kind: v3.KindGlobalNetworkPolicy, Tier: "perimeter", Policy: tgt.Policy, Index: tgt.RuleIndex, Action: "allow"}} + for name, f := range map[string]*Flow{ + "tail port": NewFlow(SourceIP, DefaultSourcePort, tgt.AddrInCIDR, int(tgt.TailPort)), + "popular port": NewFlow(SourceIP, DefaultSourcePort, tgt.AddrInCIDR, int(tgt.PopularPort)), + "aimed": fx.MatchingFlow(Egress, tgt.Ordinal), + } { + if got := fx.Expect(Egress, f); !equalVerdicts(got, want) { + t.Errorf("%s flow %v: got %v, want %v", name, f, got, want) + } + } + + denied := fx.Expect(Egress, fx.DeniedFlow(Egress)) + wantDenied := []Verdict{{Kind: v3.KindGlobalNetworkPolicy, Tier: "perimeter", Policy: "egress-000", Index: TierDefaultIndex, Action: "deny"}} + if !equalVerdicts(denied, wantDenied) { + t.Errorf("denied flow: got %v, want %v", denied, wantDenied) + } + + // A flow aimed at any rule is matched by that rule or an earlier one, never a later one. + for _, ordinal := range []int{0, 1, 4000, 18661} { + f := fx.MatchingFlow(Egress, ordinal) + got := fx.Expect(Egress, f) + if len(got) == 0 || got[0].Index == TierDefaultIndex { + t.Errorf("flow aimed at rule %d was denied: %v", ordinal, got) + continue + } + if pos := egressOrdinal(got[0]); pos > ordinal { + t.Errorf("flow aimed at rule %d matched later rule %d (%v)", ordinal, pos, got[0]) + } + } +} + +func TestBaselineOracle(t *testing.T) { + fx := Build(Baseline()) + denied := fx.Expect(Ingress, fx.DeniedFlow(Ingress)) + wantDenied := []Verdict{{Kind: v3.KindGlobalNetworkPolicy, Tier: "perimeter", Policy: "policy-000", Index: TierDefaultIndex, Action: "deny"}} + if !equalVerdicts(denied, wantDenied) { + t.Errorf("denied flow: got %v, want %v", denied, wantDenied) + } + + // The set-guarded rules match a flow drawn from their set; the port-guarded ones never do, + // so a flow aimed at one of those is matched by the first port-guarded rule instead. + passes, finals := 0, 0 + for ordinal := 0; ordinal < fx.Rules(Ingress); ordinal += 97 { + f := fx.MatchingFlow(Ingress, ordinal) + got := fx.Expect(Ingress, f) + if len(got) == 0 { + t.Fatalf("empty trace for %v", f) + } + if got[0].Index == TierDefaultIndex { + t.Errorf("flow aimed at rule %d was denied: %v", ordinal, got) + continue + } + if pos := baselineOrdinal(got[0]); pos > ordinal { + t.Errorf("flow aimed at rule %d matched later rule %d (%v)", ordinal, pos, got[0]) + } + switch got[0].Action { + case "pass": + passes++ + // A pass with no further tier and no profiles ends in the profile deny. + if len(got) != 2 || got[1].Kind != v3.KindProfile || got[1].Action != "deny" { + t.Errorf("pass trace for rule %d: %v", ordinal, got) + } + default: + finals++ + if len(got) != 1 { + t.Errorf("final trace for rule %d: %v", ordinal, got) + } + } + } + if passes == 0 { + t.Error("no aimed flow reached a pass rule") + } +} + +func TestSampler(t *testing.T) { + fx := Build(Composite()) + s := fx.NewSampler(1, FlowModel{Direction: Egress, MissFraction: 0.2, RepeatFraction: 0.3}) + seen := map[string]int{} + denied := 0 + for i := 0; i < 2000; i++ { + f := s.Next() + if f.Protocol != ProtocolTCP || f.SrcPort < 32768 { + t.Fatalf("unexpected flow %v", f) + } + key := fmt.Sprintf("%s>%s:%d", f.SrcIP, f.DstIP, f.DstPort) + seen[key]++ + if v := fx.Expect(Egress, f); v[0].Index == TierDefaultIndex { + denied++ + } + } + if denied < 200 || denied > 1000 { + t.Errorf("denied flows: %d of 2000, want roughly 20%% plus their repeats", denied) + } + repeats := 0 + for _, n := range seen { + repeats += n - 1 + } + if repeats < 300 { + t.Errorf("repeated flows: %d of 2000, want roughly 30%%", repeats) + } +} + +func TestResourcesValidateAndRoundTrip(t *testing.T) { + fx := Build(Composite()) + opts := ResourceOptions{Selector: "policyscale.projectcalico.org/target == 'true'"} + objs := fx.Resources(opts) + if got := countKind(objs, v3.KindTier); got != 1 { + t.Errorf("tiers: %d", got) + } + if got := countKind(objs, v3.KindGlobalNetworkPolicy); got != 294+301 { + t.Errorf("policies: %d", got) + } + if got := countKind(objs, v3.KindGlobalNetworkSet); got != 3709+3862 { + t.Errorf("sets: %d", got) + } + + // Validate a sample through the API validator: the tier, the first policies of each direction, + // the target policy, and a few sets including the largest. + validated := 0 + for _, obj := range objs { + validate := false + switch o := obj.(type) { + case *v3.Tier: + validate = true + case *v3.GlobalNetworkPolicy: + validate = strings.HasSuffix(o.Name, "-000") || strings.HasSuffix(o.Name, "-001") || o.Name == "perimeter."+fx.EgressTarget().Policy + case *v3.GlobalNetworkSet: + validate = o.Name == SentinelIPSetID || strings.HasSuffix(o.Name, "-0000") || len(o.Spec.Nets) > 50000 + } + if !validate { + continue + } + if err := validator.Validate(obj); err != nil { + t.Errorf("%T %s: %v", obj, obj.(interface{ GetName() string }).GetName(), err) + } + validated++ + } + if validated < 8 { + t.Errorf("validated only %d objects", validated) + } + + docs := bytes.Split(render(t, fx, opts), []byte("\n---\n")) + if len(docs) != len(objs) { + t.Fatalf("yaml documents: got %d, want %d", len(docs), len(objs)) + } + var gnp v3.GlobalNetworkPolicy + if err := yaml.Unmarshal(docs[1], &gnp); err != nil { + t.Fatal(err) + } + if gnp.Name != "perimeter.policy-000" || gnp.Spec.Tier != "perimeter" || gnp.Spec.Selector != opts.Selector { + t.Errorf("first policy: %+v", gnp.ObjectMeta) + } + if len(gnp.Spec.Types) != 1 || gnp.Spec.Types[0] != v3.PolicyTypeIngress || len(gnp.Spec.Ingress) != 68 { + t.Errorf("first policy rules: types=%v ingress=%d", gnp.Spec.Types, len(gnp.Spec.Ingress)) + } + for i, r := range gnp.Spec.Ingress { + if len(r.Destination.Ports) > 0 && (r.Protocol == nil || r.Protocol.String() != "TCP") { + t.Errorf("rule %d has ports but protocol %v", i, r.Protocol) + } + } +} + +func render(t *testing.T, fx *Fixture, opts ...ResourceOptions) []byte { + t.Helper() + var o ResourceOptions + if len(opts) > 0 { + o = opts[0] + } + var buf bytes.Buffer + if err := fx.WriteYAML(&buf, o); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func countKind(objs []runtime.Object, kind string) int { + n := 0 + for _, o := range objs { + if o.GetObjectKind().GroupVersionKind().Kind == kind { + n++ + } + } + return n +} + +func equalVerdicts(a, b []Verdict) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// egressOrdinal and baselineOrdinal recover a rule's position in the walk from its verdict, using +// the presets' policy naming and rule counts. +func egressOrdinal(v Verdict) int { + var n int + fmt.Sscanf(v.Policy, "egress-%d", &n) + return n*62 + v.Index +} + +func baselineOrdinal(v Verdict) int { + var n int + fmt.Sscanf(v.Policy, "policy-%d", &n) + return n*68 + v.Index +} diff --git a/app-policy/policyscale/resources.go b/app-policy/policyscale/resources.go new file mode 100644 index 00000000000..51514ca6187 --- /dev/null +++ b/app-policy/policyscale/resources.go @@ -0,0 +1,177 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policyscale + +import ( + "fmt" + "io" + + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/api/pkg/lib/numorstring" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/yaml" +) + +// ResourceOptions control how a fixture is rendered as Calico resources. +type ResourceOptions struct { + // Selector chooses the endpoints the policies apply to. Default: all(). + Selector string + // SetLabelKey is the label that ties a GlobalNetworkSet to the rules that select it. + // Default: policyscale.projectcalico.org/set. + SetLabelKey string + // TierOrder is the order of the generated tier(s); consecutive tiers count up from it. + // Default 100. + TierOrder float64 +} + +const ( + defaultSelector = "all()" + defaultSetLabelKey = "policyscale.projectcalico.org/set" + defaultTierOrder = 100 + apiVersion = v3.GroupVersionCurrent +) + +func (o *ResourceOptions) defaults() { + if o.Selector == "" { + o.Selector = defaultSelector + } + if o.SetLabelKey == "" { + o.SetLabelKey = defaultSetLabelKey + } + if o.TierOrder == 0 { + o.TierOrder = defaultTierOrder + } +} + +// Resources renders the fixture as the Calico resources that apply the same policy set on a +// cluster: one Tier per generated tier, one GlobalNetworkPolicy per policy (named tier.policy, in +// tier order), and one GlobalNetworkSet per IP set the store holds, labelled so that the rules' +// selectors pick it. Sets marked missing are rendered too: on a cluster nothing is ever "not +// found", and leaving them out would change the verdicts. +// +// Two departures from the proto the engine sees, both forced by the v3 API: a rule with ports +// carries protocol TCP, since ports require a protocol (every generated flow is TCP, so verdicts +// are unchanged), and policy names carry the tier prefix. +func (fx *Fixture) Resources(opts ResourceOptions) []runtime.Object { + opts.defaults() + var out []runtime.Object + + for i, t := range fx.tiers { + order := opts.TierOrder + float64(i) + action := v3.Action(t.defaultAction) + out = append(out, &v3.Tier{ + TypeMeta: metav1.TypeMeta{Kind: v3.KindTier, APIVersion: apiVersion}, + ObjectMeta: metav1.ObjectMeta{Name: t.name}, + Spec: v3.TierSpec{Order: &order, DefaultAction: &action}, + }) + } + + for _, t := range fx.tiers { + policyOrder := float64(0) + for dir, policies := range t.policies { + for _, p := range policies { + gnp := &v3.GlobalNetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: v3.KindGlobalNetworkPolicy, APIVersion: apiVersion}, + ObjectMeta: metav1.ObjectMeta{Name: t.name + "." + p.name}, + } + order := policyOrder + policyOrder++ + gnp.Spec = v3.GlobalNetworkPolicySpec{ + Tier: t.name, + Order: &order, + Selector: opts.Selector, + } + rules := make([]v3.Rule, len(p.rules)) + for i, r := range p.rules { + rules[i] = r.resource(opts.SetLabelKey) + } + if Direction(dir) == Egress { + gnp.Spec.Types = []v3.PolicyType{v3.PolicyTypeEgress} + gnp.Spec.Egress = rules + } else { + gnp.Spec.Types = []v3.PolicyType{v3.PolicyTypeIngress} + gnp.Spec.Ingress = rules + } + out = append(out, gnp) + } + } + } + + for _, id := range fx.setOrder { + s := fx.sets[id] + out = append(out, &v3.GlobalNetworkSet{ + TypeMeta: metav1.TypeMeta{Kind: v3.KindGlobalNetworkSet, APIVersion: apiVersion}, + ObjectMeta: metav1.ObjectMeta{ + Name: id, + Labels: map[string]string{opts.SetLabelKey: id}, + }, + Spec: v3.GlobalNetworkSetSpec{Nets: s.members}, + }) + } + return out +} + +func (r *ruleModel) resource(setLabelKey string) v3.Rule { + rule := v3.Rule{Action: v3.Action(capitalise(r.action))} + if len(r.dstPorts) > 0 { + tcp := numorstring.ProtocolFromString("TCP") + rule.Protocol = &tcp + for _, p := range r.dstPorts { + rule.Destination.Ports = append(rule.Destination.Ports, numorstring.SinglePort(uint16(p))) + } + } + if r.dstNet.IsValid() { + rule.Destination.Nets = []string{r.dstNet.String()} + } + rule.Source.Selector = setSelector(setLabelKey, r.srcSet) + rule.Source.NotSelector = setSelector(setLabelKey, r.notSrcSet) + rule.Destination.Selector = setSelector(setLabelKey, r.dstSet) + rule.Destination.NotSelector = setSelector(setLabelKey, r.notDstSet) + return rule +} + +func setSelector(key, setID string) string { + if setID == "" { + return "" + } + return fmt.Sprintf("%s == '%s'", key, setID) +} + +func capitalise(action string) string { + if action == "" { + return "" + } + return string(action[0]-'a'+'A') + action[1:] +} + +// WriteYAML writes Resources as a multi-document YAML stream, ready for kubectl apply -f. +func (fx *Fixture) WriteYAML(w io.Writer, opts ResourceOptions) error { + for i, obj := range fx.Resources(opts) { + if i > 0 { + if _, err := io.WriteString(w, "---\n"); err != nil { + return err + } + } + b, err := yaml.Marshal(obj) + if err != nil { + return err + } + if _, err := w.Write(b); err != nil { + return err + } + } + return nil +} diff --git a/app-policy/policyscale/spec.go b/app-policy/policyscale/spec.go new file mode 100644 index 00000000000..62be3341933 --- /dev/null +++ b/app-policy/policyscale/spec.go @@ -0,0 +1,179 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policyscale + +// Direction of the rules a policy carries, and of an evaluation. +type Direction int + +const ( + Ingress Direction = iota + Egress +) + +func (d Direction) String() string { + if d == Egress { + return "egress" + } + return "ingress" +} + +// Spec describes a synthetic policy set applied to one endpoint. A nil preset is left out. +type Spec struct { + // Seed drives every random choice. Two Specs that are equal build identical fixtures. + Seed int64 + Baseline *BaselineParams + Egress *EgressParams +} + +// DefaultSeed is the seed the presets use; it is the one the original benchmarks were written +// with, so their numbers stay comparable. +const DefaultSeed = 20200 + +// SizeBucket is one bucket of an IP set size histogram: Sets sets of between MinSize and MaxSize +// members each. +type SizeBucket struct { + Sets, MinSize, MaxSize int +} + +// BaselineParams describes a policy store dominated by "baseline" policies: every policy applies +// to the endpoint, and almost every rule is a Pass. The rules are ingress rules. +type BaselineParams struct { + Tier string + Policies int + RulesPerPolicy int + // DenyRules and AllowRules are spread at random over the rule slots; every other rule is a Pass. + DenyRules int + AllowRules int + // IPSetRefFraction is the fraction of rules that reference an IP set. A referencing rule pairs + // the reference with a negated reference to the sentinel set (which holds the denied flow's + // addresses) so that it reaches the set lookup whatever order the engine evaluates criteria in + // and still misses when the referenced set is absent. Rules with no reference are guarded by a + // destination port no generated flow uses (BaselineGuardPort). + IPSetRefFraction float64 + // MissingIPSets is how many referenced sets to leave out of the store, reproducing the + // "IPSet not found" warning storm seen when the store is out of sync. + MissingIPSets int + // SizeHistogram is the IP set size distribution. Members are unique /32s from 10.0.0.0/9. + SizeHistogram []SizeBucket +} + +// PortWeight is the fraction of egress rules whose destination ports include Port. +type PortWeight struct { + Port int32 + Fraction float64 +} + +// PortCount is the fraction of egress rules that carry Count destination ports. +type PortCount struct { + Count int + Fraction float64 +} + +// EgressParams describes a single-tier destination allow-list: every policy applies to the +// endpoint, every rule is a Pass matching a destination address plus a handful of destination +// ports, and a flow that matches nothing meets the tier default deny. +type EgressParams struct { + Tier string + Policies int + RulesPerPolicy int + // IPSetRuleFraction is the fraction of rules whose destination is an IP set (a selector, once + // it reaches the engine); the rest carry a CIDR. CIDRs are handed out sequentially from + // 10.0.0.0/9 so that no two rules share one. + IPSetRuleFraction float64 + // IPSets and MembersPerSet size the destination sets. Members are unique /32s from + // 10.128.0.0/9, disjoint from the rule CIDRs. + IPSets int + MembersPerSet int + // PortWeights is the head of the destination port distribution; PortsPerRule the spread of + // port counts per rule. Rules top up from a long tail of otherwise-unique ports. + PortWeights []PortWeight + PortsPerRule []PortCount + // TargetDepth, when positive, turns the rule at this fraction of the walk into an Allow with a + // CIDR and a port of its own, so that a flow aimed at it matches nothing earlier. The + // benchmarks use it to measure a match at a known depth; see Fixture.EgressTarget. + TargetDepth float64 +} + +// DefaultBaseline returns the anonymised per-node scale measured in a large production +// deployment: 294 policies of 68 rules, ~24% referencing one of 3,708 IP sets, dominated by tiny +// sets with a long tail of large ones. The measured sets held ~256k members in total; drawn +// uniformly within each bucket the histogram gives about 1M, which the tests pin. +func DefaultBaseline() BaselineParams { + return BaselineParams{ + Tier: "perimeter", + Policies: 294, + RulesPerPolicy: 68, + DenyRules: 10, + AllowRules: 1, + IPSetRefFraction: 0.242, // ~4,838 IP set references across 294*68 rules. + SizeHistogram: []SizeBucket{ + {4, 0, 0}, + {3035, 1, 9}, + {446, 10, 99}, + {64, 100, 999}, + {158, 1000, 9999}, + {1, 54566, 54566}, + }, + } +} + +// DefaultEgress returns the second rule-set shape measured in production: 301 policies of 62 +// egress rules (18,662 in all), 4,644 of 18,675 measured rules with a selector destination, and +// the measured port distribution (443 on 18% of rules, 11001 and 27054 on 12% each, 80 on 8%; +// median 2 ports per rule, mean ~5). The target sits at 65% of the walk. +func DefaultEgress() EgressParams { + return EgressParams{ + Tier: "perimeter", + Policies: 301, + RulesPerPolicy: 62, + IPSetRuleFraction: 0.25, + IPSets: 3862, + MembersPerSet: 3, + PortWeights: []PortWeight{ + {443, 0.182}, + {11001, 0.123}, + {27054, 0.122}, + {80, 0.081}, + }, + PortsPerRule: []PortCount{ + {1, 0.25}, + {2, 0.45}, + {3, 0.15}, + {8, 0.10}, + {20, 0.05}, + }, + TargetDepth: 0.65, + } +} + +// Baseline is the baseline preset on its own. +func Baseline() Spec { + p := DefaultBaseline() + return Spec{Seed: DefaultSeed, Baseline: &p} +} + +// EgressAllowList is the egress allow-list preset on its own. +func EgressAllowList() Spec { + p := DefaultEgress() + return Spec{Seed: DefaultSeed, Egress: &p} +} + +// Composite applies both presets to one endpoint: the baseline set governs its ingress +// evaluations and the allow-list its egress evaluations. This is the reference set the +// PMREQ-954 targets are quoted against. +func Composite() Spec { + b, e := DefaultBaseline(), DefaultEgress() + return Spec{Seed: DefaultSeed, Baseline: &b, Egress: &e} +} diff --git a/app-policy/policyscale/store.go b/app-policy/policyscale/store.go new file mode 100644 index 00000000000..17fbc386c25 --- /dev/null +++ b/app-policy/policyscale/store.go @@ -0,0 +1,97 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policyscale + +import ( + "github.com/projectcalico/calico/app-policy/policystore" + "github.com/projectcalico/calico/felix/proto" + "github.com/projectcalico/calico/felix/types" +) + +// NewStore returns a fresh policy store holding the fixture's policies and the IP sets that are +// not marked missing. Policies are shared between stores built from the same fixture; sets are +// built per store, so a test may mutate them. +func (fx *Fixture) NewStore() *policystore.PolicyStore { + store := policystore.NewPolicyStore() + fx.LoadStore(store) + return store +} + +// LoadStore adds the fixture's policies and IP sets to an existing store, for callers that own +// the store's lifecycle (a policystore.PolicyStoreManager, say). +func (fx *Fixture) LoadStore(store *policystore.PolicyStore) { + for _, id := range fx.setOrder { + s := fx.sets[id] + if s.missing { + continue + } + set := policystore.NewIPSet(proto.IPSetUpdate_NET) + for _, m := range s.members { + set.AddString(m) + } + store.IPSetByID[id] = set + } + for _, t := range fx.tiers { + for _, policies := range t.policies { + for _, p := range policies { + store.PolicyByID[types.ProtoToPolicyID(p.id)] = p.policy + } + } + } +} + +// Updates returns the fixture as the dataplane messages Felix would send to load it into a store +// through PolicyStore.ProcessUpdate: IP sets first, then policies, as Felix orders them. The +// endpoint is not included; callers add it under the ID of their choice. +func (fx *Fixture) Updates() []*proto.ToDataplane { + var updates []*proto.ToDataplane + for _, id := range fx.setOrder { + s := fx.sets[id] + if s.missing { + continue + } + updates = append(updates, &proto.ToDataplane{Payload: &proto.ToDataplane_IpsetUpdate{ + IpsetUpdate: &proto.IPSetUpdate{Id: id, Type: proto.IPSetUpdate_NET, Members: s.members}, + }}) + } + for _, t := range fx.tiers { + for _, policies := range t.policies { + for _, p := range policies { + updates = append(updates, &proto.ToDataplane{Payload: &proto.ToDataplane_ActivePolicyUpdate{ + ActivePolicyUpdate: &proto.ActivePolicyUpdate{Id: p.id, Policy: p.policy}, + }}) + } + } + } + return updates +} + +// Endpoint returns a workload endpoint to which every policy of the fixture applies, with one +// TierInfo per tier carrying the ingress and egress policy lists. Each call returns a new +// endpoint, so a test may prepend a policy without disturbing other users of the fixture. +func (fx *Fixture) Endpoint() *proto.WorkloadEndpoint { + ep := &proto.WorkloadEndpoint{State: "active", Name: "policyscale0"} + for _, t := range fx.tiers { + ti := &proto.TierInfo{Name: t.name, DefaultAction: t.defaultAction} + for _, p := range t.policies[Ingress] { + ti.IngressPolicies = append(ti.IngressPolicies, p.id) + } + for _, p := range t.policies[Egress] { + ti.EgressPolicies = append(ti.EgressPolicies, p.id) + } + ep.Tiers = append(ep.Tiers, ti) + } + return ep +} diff --git a/app-policy/policystore/process.go b/app-policy/policystore/process.go index 0836a1932b2..9a29cb400b0 100644 --- a/app-policy/policystore/process.go +++ b/app-policy/policystore/process.go @@ -26,6 +26,10 @@ import ( // Staged policies are stored like any other. Whether they count towards a verdict is up to the // evaluation, which takes a checker.PolicyScope; see checker.Evaluate. func (store *PolicyStore) ProcessUpdate(subscriptionType string, update *proto.ToDataplane) { + // Every update but InSync can change a verdict; see Generation. + if _, inSync := update.Payload.(*proto.ToDataplane_InSync); !inSync { + store.Generation++ + } // TODO: maybe coalesce-ing updater fits here switch payload := update.Payload.(type) { case *proto.ToDataplane_InSync: diff --git a/app-policy/policystore/store.go b/app-policy/policystore/store.go index 9f09731c8e8..549adb25fbf 100644 --- a/app-policy/policystore/store.go +++ b/app-policy/policystore/store.go @@ -42,6 +42,16 @@ type PolicyStore struct { Endpoints map[types.WorkloadEndpointID]*proto.WorkloadEndpoint ServiceAccountByID map[types.ServiceAccountID]*proto.ServiceAccountUpdate NamespaceByID map[types.NamespaceID]*proto.NamespaceUpdate + + // Generation counts the updates ProcessUpdate has applied to the store. State derived from the + // store's contents (the verdict cache) records the generation it was derived at and is discarded + // when the generation moves on. Code that mutates the store other than through ProcessUpdate + // must increment it. + Generation uint64 + // Verdicts caches evaluation results for this store's contents; nil when caching is off. Set + // through WithVerdictCache on the manager that creates the store, so that a cache never outlives + // the store it was filled from. + Verdicts *VerdictCache } func NewPolicyStore() *PolicyStore { @@ -61,6 +71,8 @@ type policyStoreManager struct { current, pending *PolicyStore mu sync.RWMutex toActive bool + // newStore creates the stores the manager hands out: at construction and on every reconnect. + newStore func() *PolicyStore } type PolicyStoreManager interface { @@ -86,16 +98,27 @@ func NewPolicyStoreManager() PolicyStoreManager { } func NewPolicyStoreManagerWithOpts(opts ...PolicyStoreManagerOption) *policyStoreManager { - psm := &policyStoreManager{ - current: NewPolicyStore(), - pending: NewPolicyStore(), - } + psm := &policyStoreManager{newStore: NewPolicyStore} for _, o := range opts { o(psm) } + psm.current = psm.newStore() + psm.pending = psm.newStore() return psm } +// WithVerdictCache gives every store the manager creates a verdict cache of the given capacity, +// reporting into the shared counters. See VerdictCache for what it caches and when it is emptied. +func WithVerdictCache(capacity int, stats *VerdictCacheStats) PolicyStoreManagerOption { + return func(m *policyStoreManager) { + m.newStore = func() *PolicyStore { + s := NewPolicyStore() + s.Verdicts = NewVerdictCache(capacity, stats) + return s + } + } +} + func (m *policyStoreManager) DoWithReadLock(cb func(*PolicyStore)) { log.Tracef("StoreManager acquiring read lock") m.mu.RLock() @@ -140,7 +163,7 @@ func (m *policyStoreManager) OnReconnecting() { defer m.mu.Unlock() // create store - m.pending = NewPolicyStore() + m.pending = m.newStore() log.Tracef("storeManager OnReconnecting() created new pending store %p", m.pending) // route next writes to pending diff --git a/app-policy/policystore/verdictcache.go b/app-policy/policystore/verdictcache.go new file mode 100644 index 00000000000..faf158bd172 --- /dev/null +++ b/app-policy/policystore/verdictcache.go @@ -0,0 +1,158 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policystore + +import ( + "sync" + "sync/atomic" + + "github.com/projectcalico/calico/felix/proto" +) + +// VerdictKey identifies a cached verdict: the endpoint evaluated, the scope and direction of the +// evaluation, and the parts of the flow's L3/L4 header that the endpoint's rules can look at. The +// source port is part of the key only when some applicable rule matches on source ports; flows +// that differ only in their source port (a client reconnecting) then share one entry. The checker +// decides that; see checker.verdictKey. +type VerdictKey struct { + Endpoint *proto.WorkloadEndpoint + Scope int8 + Direction int8 + Protocol int32 + SrcPort int32 // -1 when the key does not include the source port. + DstPort int32 + SrcIP [16]byte + DstIP [16]byte +} + +// VerdictCacheStats counts cache traffic. One set of counters is shared between the caches of +// successive stores, so that they survive a resync. +type VerdictCacheStats struct { + Hits, Misses, Resets, Evictions atomic.Uint64 +} + +// VerdictCache remembers evaluation results for the contents of one PolicyStore. +// +// Entries are valid for one store generation: every Lookup and Store carries the store's current +// Generation, and a cache that sees a new generation starts empty. Any update applied through +// ProcessUpdate moves the generation, so a cached verdict can never outlive the policies, IP sets +// or endpoints it was computed from. This is coarse (an IP set delta anywhere empties the cache) +// and safe; finer invalidation is a separate piece of work. +// +// The cache is bounded. When it is full it starts over rather than evicting one entry: an LRU +// would keep the hot entries but costs a list operation per lookup, and the flows the collector +// sees repeat within a window far smaller than the default capacity. +// +// The cache holds its own lock, so it is safe to use from concurrent evaluations holding the +// store's read lock. +type VerdictCache struct { + mu sync.Mutex + capacity int + generation uint64 + entries map[VerdictKey]any + flags map[flagKey]bool + stats *VerdictCacheStats +} + +// flagKey identifies a per-endpoint property memoised for one generation. +type flagKey struct { + ep *proto.WorkloadEndpoint + scope, dir int8 +} + +// NewVerdictCache returns a cache holding at most capacity entries, reporting into stats (a +// private set of counters when nil). +func NewVerdictCache(capacity int, stats *VerdictCacheStats) *VerdictCache { + if stats == nil { + stats = &VerdictCacheStats{} + } + return &VerdictCache{ + capacity: capacity, + entries: make(map[VerdictKey]any), + flags: make(map[flagKey]bool), + stats: stats, + } +} + +// Lookup returns the verdict cached for the key at this generation, if any. +func (c *VerdictCache) Lookup(generation uint64, key VerdictKey) (any, bool) { + c.mu.Lock() + defer c.mu.Unlock() + c.syncGeneration(generation) + v, ok := c.entries[key] + if ok { + c.stats.Hits.Add(1) + } else { + c.stats.Misses.Add(1) + } + return v, ok +} + +// Store records a verdict for the key at this generation. The verdict is handed out as-is to +// later lookups, so it must not be modified afterwards. +func (c *VerdictCache) Store(generation uint64, key VerdictKey, verdict any) { + c.mu.Lock() + defer c.mu.Unlock() + c.syncGeneration(generation) + if c.capacity <= 0 { + return + } + if len(c.entries) >= c.capacity { + clear(c.entries) + c.stats.Evictions.Add(1) + } + c.entries[key] = verdict +} + +// EndpointFlag memoises, for the current generation, a property of the rules that apply to an +// endpoint in a scope and direction, computing it on first use. The checker uses it to decide how +// the endpoint's flows are keyed without walking the rules on every evaluation. +func (c *VerdictCache) EndpointFlag(generation uint64, ep *proto.WorkloadEndpoint, scope, dir int8, compute func() bool) bool { + c.mu.Lock() + defer c.mu.Unlock() + c.syncGeneration(generation) + k := flagKey{ep: ep, scope: scope, dir: dir} + if v, ok := c.flags[k]; ok { + return v + } + v := compute() + c.flags[k] = v + return v +} + +// syncGeneration empties the cache when the store has moved on. Called with the lock held. +func (c *VerdictCache) syncGeneration(generation uint64) { + if c.generation == generation { + return + } + if len(c.entries) > 0 || len(c.flags) > 0 { + clear(c.entries) + clear(c.flags) + c.stats.Resets.Add(1) + } + c.generation = generation +} + +// Len returns the number of cached verdicts. +func (c *VerdictCache) Len() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.entries) +} + +// Stats returns the counters the cache reports into. +func (c *VerdictCache) Stats() *VerdictCacheStats { + return c.stats +} diff --git a/app-policy/policystore/verdictcache_test.go b/app-policy/policystore/verdictcache_test.go new file mode 100644 index 00000000000..134eeac16ee --- /dev/null +++ b/app-policy/policystore/verdictcache_test.go @@ -0,0 +1,161 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policystore + +import ( + "testing" + + "github.com/projectcalico/calico/felix/proto" +) + +func TestProcessUpdateBumpsGeneration(t *testing.T) { + store := NewPolicyStore() + wepID := &proto.WorkloadEndpointID{OrchestratorId: "k8s", WorkloadId: "ns/pod", EndpointId: "eth0"} + updates := []*proto.ToDataplane{ + {Payload: &proto.ToDataplane_IpsetUpdate{IpsetUpdate: &proto.IPSetUpdate{Id: "s", Type: proto.IPSetUpdate_NET}}}, + {Payload: &proto.ToDataplane_IpsetDeltaUpdate{IpsetDeltaUpdate: &proto.IPSetDeltaUpdate{Id: "s", AddedMembers: []string{"10.0.0.1/32"}}}}, + {Payload: &proto.ToDataplane_ActivePolicyUpdate{ActivePolicyUpdate: &proto.ActivePolicyUpdate{Id: &proto.PolicyID{Name: "p"}, Policy: &proto.Policy{}}}}, + {Payload: &proto.ToDataplane_ActivePolicyRemove{ActivePolicyRemove: &proto.ActivePolicyRemove{Id: &proto.PolicyID{Name: "p"}}}}, + {Payload: &proto.ToDataplane_ActiveProfileUpdate{ActiveProfileUpdate: &proto.ActiveProfileUpdate{Id: &proto.ProfileID{Name: "pr"}, Profile: &proto.Profile{}}}}, + {Payload: &proto.ToDataplane_ActiveProfileRemove{ActiveProfileRemove: &proto.ActiveProfileRemove{Id: &proto.ProfileID{Name: "pr"}}}}, + {Payload: &proto.ToDataplane_WorkloadEndpointUpdate{WorkloadEndpointUpdate: &proto.WorkloadEndpointUpdate{Id: wepID, Endpoint: &proto.WorkloadEndpoint{}}}}, + {Payload: &proto.ToDataplane_WorkloadEndpointRemove{WorkloadEndpointRemove: &proto.WorkloadEndpointRemove{Id: wepID}}}, + {Payload: &proto.ToDataplane_ServiceAccountUpdate{ServiceAccountUpdate: &proto.ServiceAccountUpdate{Id: &proto.ServiceAccountID{Name: "sa", Namespace: "ns"}}}}, + {Payload: &proto.ToDataplane_ServiceAccountRemove{ServiceAccountRemove: &proto.ServiceAccountRemove{Id: &proto.ServiceAccountID{Name: "sa", Namespace: "ns"}}}}, + {Payload: &proto.ToDataplane_NamespaceUpdate{NamespaceUpdate: &proto.NamespaceUpdate{Id: &proto.NamespaceID{Name: "ns"}}}}, + {Payload: &proto.ToDataplane_NamespaceRemove{NamespaceRemove: &proto.NamespaceRemove{Id: &proto.NamespaceID{Name: "ns"}}}}, + {Payload: &proto.ToDataplane_IpsetRemove{IpsetRemove: &proto.IPSetRemove{Id: "s"}}}, + } + for i, u := range updates { + store.ProcessUpdate("per-host-policies", u) + if store.Generation != uint64(i+1) { + t.Fatalf("after update %d (%T): generation %d", i, u.Payload, store.Generation) + } + } + store.ProcessUpdate("per-host-policies", &proto.ToDataplane{Payload: &proto.ToDataplane_InSync{InSync: &proto.InSync{}}}) + if store.Generation != uint64(len(updates)) { + t.Fatalf("InSync moved the generation to %d", store.Generation) + } +} + +func TestVerdictCacheGenerationAndCapacity(t *testing.T) { + stats := &VerdictCacheStats{} + c := NewVerdictCache(2, stats) + k1, k2, k3 := VerdictKey{DstPort: 1}, VerdictKey{DstPort: 2}, VerdictKey{DstPort: 3} + + if _, ok := c.Lookup(1, k1); ok { + t.Fatal("hit on an empty cache") + } + c.Store(1, k1, "a") + if v, ok := c.Lookup(1, k1); !ok || v != "a" { + t.Fatalf("lookup after store: %v %v", v, ok) + } + if stats.Hits.Load() != 1 || stats.Misses.Load() != 1 { + t.Fatalf("stats after one miss and one hit: %+v", stats) + } + + // At capacity the cache starts over. + c.Store(1, k2, "b") + if c.Len() != 2 { + t.Fatalf("len %d, want 2", c.Len()) + } + c.Store(1, k3, "c") + if c.Len() != 1 || stats.Evictions.Load() != 1 { + t.Fatalf("after eviction: len %d evictions %d", c.Len(), stats.Evictions.Load()) + } + if _, ok := c.Lookup(1, k1); ok { + t.Fatal("k1 survived the eviction") + } + if v, ok := c.Lookup(1, k3); !ok || v != "c" { + t.Fatal("k3 was not kept") + } + + // A new generation empties it. + if _, ok := c.Lookup(2, k3); ok { + t.Fatal("entry survived a generation change") + } + if c.Len() != 0 || stats.Resets.Load() != 1 { + t.Fatalf("after generation change: len %d resets %d", c.Len(), stats.Resets.Load()) + } + // Seeing the same new generation again is not a reset. + c.Store(2, k1, "a") + if _, ok := c.Lookup(2, k1); !ok || stats.Resets.Load() != 1 { + t.Fatalf("second use of generation 2 reset the cache: resets %d", stats.Resets.Load()) + } + + // A zero-capacity cache stores nothing. + z := NewVerdictCache(0, nil) + z.Store(1, k1, "a") + if _, ok := z.Lookup(1, k1); ok || z.Len() != 0 { + t.Fatal("zero-capacity cache stored an entry") + } +} + +func TestVerdictCacheEndpointFlag(t *testing.T) { + stats := &VerdictCacheStats{} + c := NewVerdictCache(8, stats) + ep, other := &proto.WorkloadEndpoint{Name: "a"}, &proto.WorkloadEndpoint{Name: "b"} + calls := 0 + compute := func() bool { calls++; return calls%2 == 1 } + + if !c.EndpointFlag(1, ep, 0, 0, compute) || calls != 1 { + t.Fatalf("first computation: calls %d", calls) + } + if !c.EndpointFlag(1, ep, 0, 0, compute) || calls != 1 { + t.Fatalf("memoised value not returned: calls %d", calls) + } + // Direction, scope and endpoint are part of the memo key. + c.EndpointFlag(1, ep, 0, 1, compute) + c.EndpointFlag(1, ep, 1, 0, compute) + c.EndpointFlag(1, other, 0, 0, compute) + if calls != 4 { + t.Fatalf("expected one computation per key, got %d", calls) + } + // A new generation recomputes. + c.EndpointFlag(2, ep, 0, 0, compute) + if calls != 5 || stats.Resets.Load() != 1 { + t.Fatalf("after generation change: calls %d resets %d", calls, stats.Resets.Load()) + } +} + +func TestWithVerdictCacheOnEveryStore(t *testing.T) { + stats := &VerdictCacheStats{} + m := NewPolicyStoreManagerWithOpts(WithVerdictCache(8, stats)) + var pending *PolicyStore + m.DoWithLock(func(s *PolicyStore) { + pending = s + if s.Verdicts == nil || s.Verdicts.Stats() != stats { + t.Fatal("pending store has no cache with the shared stats") + } + }) + m.OnInSync() + m.DoWithReadLock(func(s *PolicyStore) { + if s != pending || s.Verdicts == nil { + t.Fatal("current store after sync is not the cached pending store") + } + }) + m.OnReconnecting() + m.DoWithLock(func(s *PolicyStore) { + if s == pending || s.Verdicts == nil || s.Verdicts.Stats() != stats { + t.Fatal("store created on reconnect has no cache with the shared stats") + } + }) + + NewPolicyStoreManager().DoWithReadLock(func(s *PolicyStore) { + if s.Verdicts != nil { + t.Fatal("default manager created a cache") + } + }) +} diff --git a/felix/collector/collector.go b/felix/collector/collector.go index d3b53b4cd82..010bfd68bdb 100644 --- a/felix/collector/collector.go +++ b/felix/collector/collector.go @@ -115,6 +115,27 @@ var ( }, []string{"reason"}) + // verdictCacheStats is shared by the caches of every policy store the collector's manager + // creates, so the counters survive a resync. One collector per process, so package-level. + verdictCacheStats = &policystore.VerdictCacheStats{} + + counterPolicyEvalCacheHits = prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "felix_collector_policy_eval_cache_hits_total", + Help: "Total number of pending policy evaluations answered from the verdict cache.", + }, func() float64 { return float64(verdictCacheStats.Hits.Load()) }) + counterPolicyEvalCacheMisses = prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "felix_collector_policy_eval_cache_misses_total", + Help: "Total number of pending policy evaluations that missed the verdict cache and walked the policy set.", + }, func() float64 { return float64(verdictCacheStats.Misses.Load()) }) + counterPolicyEvalCacheResets = prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "felix_collector_policy_eval_cache_resets_total", + Help: "Total number of times the verdict cache was emptied because policy, IP set or endpoint state changed.", + }, func() float64 { return float64(verdictCacheStats.Resets.Load()) }) + counterPolicyEvalCacheEvictions = prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "felix_collector_policy_eval_cache_evictions_total", + Help: "Total number of times the verdict cache was emptied because it reached its capacity.", + }, func() float64 { return float64(verdictCacheStats.Evictions.Load()) }) + histogramPolicyEvalSweepDuration = prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "felix_collector_policy_eval_sweep_duration_seconds", Help: "Wall-clock time to drain one policy re-evaluation snapshot across all its batches.", @@ -130,6 +151,10 @@ func init() { prometheus.MustRegister(counterPolicyEvalBatches) prometheus.MustRegister(counterPolicyEvalFlows) prometheus.MustRegister(histogramPolicyEvalSweepDuration) + prometheus.MustRegister(counterPolicyEvalCacheHits) + prometheus.MustRegister(counterPolicyEvalCacheMisses) + prometheus.MustRegister(counterPolicyEvalCacheResets) + prometheus.MustRegister(counterPolicyEvalCacheEvictions) } type Config struct { @@ -140,6 +165,9 @@ type Config struct { EnableServices bool PolicyEvaluationMode string FlowLogsFlushInterval time.Duration + // PolicyEvaluationCacheSize is the capacity of the verdict cache in front of the pending-policy + // evaluation; 0 disables it. Only used when the collector creates its own PolicyStoreManager. + PolicyEvaluationCacheSize int IsBPFDataplane bool @@ -202,7 +230,12 @@ func newCollector(lc *calc.LookupsCache, cfg *Config) Collector { } if c.policyStoreManager == nil { - c.policyStoreManager = policystore.NewPolicyStoreManager() + var opts []policystore.PolicyStoreManagerOption + if cfg.PolicyEvaluationCacheSize > 0 { + log.Infof("Pending policy verdict cache enabled, capacity %d", cfg.PolicyEvaluationCacheSize) + opts = append(opts, policystore.WithVerdictCache(cfg.PolicyEvaluationCacheSize, verdictCacheStats)) + } + c.policyStoreManager = policystore.NewPolicyStoreManagerWithOpts(opts...) } // Only run the re-evaluation sweep when pending policies are enabled; leaving the ticker nil diff --git a/felix/collector/dpstatshelper.go b/felix/collector/dpstatshelper.go index 637b9c070f5..808807d7d2c 100644 --- a/felix/collector/dpstatshelper.go +++ b/felix/collector/dpstatshelper.go @@ -44,16 +44,17 @@ func New( statsCollector := newCollector( lookupsCache, &Config{ - AgeTimeout: config.DefaultAgeTimeout, - InitialReportingDelay: config.DefaultInitialReportingDelay, - ExportingInterval: config.DefaultExportingInterval, - EnableServices: true, - EnableNetworkSets: true, - PolicyEvaluationMode: configParams.FlowLogsPolicyEvaluationMode, - FlowLogsFlushInterval: configParams.FlowLogsFlushInterval, - IsBPFDataplane: configParams.BPFEnabled, - DisplayDebugTraceLogs: configParams.FlowLogsCollectorDebugTrace, - BPFConntrackTimeouts: bpfconntrack.GetTimeouts(configParams.BPFConntrackTimeouts), + AgeTimeout: config.DefaultAgeTimeout, + InitialReportingDelay: config.DefaultInitialReportingDelay, + ExportingInterval: config.DefaultExportingInterval, + EnableServices: true, + EnableNetworkSets: true, + PolicyEvaluationMode: configParams.FlowLogsPolicyEvaluationMode, + FlowLogsFlushInterval: configParams.FlowLogsFlushInterval, + PolicyEvaluationCacheSize: configParams.FlowLogsPolicyEvaluationCacheSize, + IsBPFDataplane: configParams.BPFEnabled, + DisplayDebugTraceLogs: configParams.FlowLogsCollectorDebugTrace, + BPFConntrackTimeouts: bpfconntrack.GetTimeouts(configParams.BPFConntrackTimeouts), }, ) diff --git a/felix/collector/verdictcache_test.go b/felix/collector/verdictcache_test.go new file mode 100644 index 00000000000..f6c450d2e62 --- /dev/null +++ b/felix/collector/verdictcache_test.go @@ -0,0 +1,42 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "testing" + "time" + + "github.com/projectcalico/calico/app-policy/policystore" +) + +func TestCollectorVerdictCacheWiring(t *testing.T) { + for _, size := range []int{0, 1024} { + c := newCollector(newMockLookupsCache(nil, nil, nil, nil), &Config{ + AgeTimeout: time.Second, + InitialReportingDelay: time.Second, + ExportingInterval: time.Second, + FlowLogsFlushInterval: time.Second, + PolicyEvaluationCacheSize: size, + }).(*collector) + c.policyStoreManager.DoWithLock(func(ps *policystore.PolicyStore) { + if (ps.Verdicts != nil) != (size > 0) { + t.Errorf("cache size %d: store cache present=%v", size, ps.Verdicts != nil) + } + if ps.Verdicts != nil && ps.Verdicts.Stats() != verdictCacheStats { + t.Errorf("cache size %d: cache does not report into the exported counters", size) + } + }) + } +} diff --git a/felix/config/config_params.go b/felix/config/config_params.go index edf47abef2e..7eb085e620e 100644 --- a/felix/config/config_params.go +++ b/felix/config/config_params.go @@ -449,6 +449,12 @@ type Config struct { FlowLogsGoldmaneServer string `config:"string;"` FlowLogsLocalReporter string `config:"oneof(Enabled,Disabled);Disabled"` FlowLogsPolicyEvaluationMode string `config:"oneof(None,Continuous);Continuous"` + // FlowLogsPolicyEvaluationCacheSize: the number of pending-policy verdicts the flow log + // collector remembers, keyed on a flow's addresses, protocol and destination port (and its + // source port only where a policy rule matches on source ports), so that a flow repeating an + // earlier flow's endpoints is answered without walking the policy set. The cache is emptied + // whenever policy, IP set or endpoint state changes. Set to 0 to disable it. + FlowLogsPolicyEvaluationCacheSize int `config:"int(0:);65536;local"` KubeNodePortRanges []numorstring.Port `config:"portrange-list;30000:32767"` NATPortRange numorstring.Port `config:"portrange;"` diff --git a/felix/docs/config-params.json b/felix/docs/config-params.json index 011419163d1..c0421fb1cbb 100644 --- a/felix/docs/config-params.json +++ b/felix/docs/config-params.json @@ -5299,6 +5299,32 @@ "UserEditable": true, "GoType": "*string" }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsPolicyEvaluationCacheSize", + "NameEnvVar": "FELIX_FlowLogsPolicyEvaluationCacheSize", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Integer: [0,2^63-1]", + "StringSchemaHTML": "Integer: [0,263-1]", + "StringDefault": "65536", + "ParsedDefault": "65536", + "ParsedDefaultJSON": "65536", + "ParsedType": "int", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "The number of pending-policy verdicts the flow log\ncollector remembers, keyed on a flow's addresses, protocol and destination port (and its\nsource port only where a policy rule matches on source ports), so that a flow repeating an\nearlier flow's endpoints is answered without walking the policy set. The cache is emptied\nwhenever policy, IP set or endpoint state changes. Set to 0 to disable it.", + "DescriptionHTML": "

The number of pending-policy verdicts the flow log\ncollector remembers, keyed on a flow's addresses, protocol and destination port (and its\nsource port only where a policy rule matches on source ports), so that a flow repeating an\nearlier flow's endpoints is answered without walking the policy set. The cache is emptied\nwhenever policy, IP set or endpoint state changes. Set to 0 to disable it.

", + "UserEditable": true, + "GoType": "" + }, { "Group": "Flow logs: file reports", "GroupWithSortPrefix": "40 Flow logs: file reports", diff --git a/felix/docs/config-params.md b/felix/docs/config-params.md index f09b141b02b..8b531a9abb6 100644 --- a/felix/docs/config-params.md +++ b/felix/docs/config-params.md @@ -2969,6 +2969,21 @@ Configures local unix socket for reporting flow data from each node. | `FelixConfiguration` schema | One of: "Disabled", "Enabled". | | Default value (YAML) | `Disabled` | +### `FlowLogsPolicyEvaluationCacheSize` (config file / env var only) + +The number of pending-policy verdicts the flow log +collector remembers, keyed on a flow's addresses, protocol and destination port (and its +source port only where a policy rule matches on source ports), so that a flow repeating an +earlier flow's endpoints is answered without walking the policy set. The cache is emptied +whenever policy, IP set or endpoint state changes. Set to 0 to disable it. + +| Detail | | +| --- | --- | +| Environment variable | `FELIX_FlowLogsPolicyEvaluationCacheSize` | +| Encoding (env var/config file) | Integer: [0,263-1] | +| Default value (above encoding) | `65536` | +| Notes | Config file / env var only. | + ### `FlowLogsPolicyEvaluationMode` (config file) / `flowLogsPolicyEvaluationMode` (YAML) Continuous - Felix evaluates active flows on a regular basis to determine the rule diff --git a/hack/cmd/policyscale/main.go b/hack/cmd/policyscale/main.go new file mode 100644 index 00000000000..02355f9baad --- /dev/null +++ b/hack/cmd/policyscale/main.go @@ -0,0 +1,98 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// policyscale renders one of the app-policy/policyscale presets as Calico resources, for applying +// the benchmark policy sets to a cluster, or prints flows drawn from its flow model with the +// verdict the engine must reach for each, for driving and checking a node-level run. +// +// go run ./hack/cmd/policyscale -preset composite > composite.yaml +// kubectl label pod flowgen-target policyscale.projectcalico.org/target=true +// kubectl apply -f composite.yaml +// +// go run ./hack/cmd/policyscale -preset composite -flows 1000 -direction egress +package main + +import ( + "bufio" + "flag" + "fmt" + "os" + + "github.com/projectcalico/calico/app-policy/policyscale" +) + +func main() { + var ( + preset = flag.String("preset", "composite", "policy set to render: baseline, egress or composite") + seed = flag.Int64("seed", policyscale.DefaultSeed, "generator seed") + selector = flag.String("selector", "policyscale.projectcalico.org/target == 'true'", "endpoint selector the policies apply to") + out = flag.String("out", "-", "output file, - for stdout") + flows = flag.Int("flows", 0, "instead of resources, print this many flows with their expected verdicts") + direction = flag.String("direction", "egress", "direction of the printed flows: ingress or egress") + miss = flag.Float64("miss-fraction", 0.1, "fraction of printed flows that match no rule") + repeat = flag.Float64("repeat-fraction", 0.5, "fraction of printed flows that repeat an earlier one with a new source port") + ) + flag.Parse() + + var spec policyscale.Spec + switch *preset { + case "baseline": + spec = policyscale.Baseline() + case "egress": + spec = policyscale.EgressAllowList() + case "composite": + spec = policyscale.Composite() + default: + fmt.Fprintf(os.Stderr, "unknown preset %q\n", *preset) + os.Exit(2) + } + spec.Seed = *seed + fx := policyscale.Build(spec) + + w := os.Stdout + if *out != "-" { + f, err := os.Create(*out) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + defer f.Close() + w = f + } + bw := bufio.NewWriter(w) + defer bw.Flush() + + fmt.Fprintf(os.Stderr, "%s: %d ingress rules in %d policies, %d egress rules in %d policies, %d IP sets with %d members\n", + *preset, fx.Rules(policyscale.Ingress), fx.Policies(policyscale.Ingress), + fx.Rules(policyscale.Egress), fx.Policies(policyscale.Egress), fx.IPSets(), fx.IPSetMembers()) + + if *flows > 0 { + dir := policyscale.Egress + if *direction == "ingress" { + dir = policyscale.Ingress + } + s := fx.NewSampler(*seed, policyscale.FlowModel{Direction: dir, MissFraction: *miss, RepeatFraction: *repeat}) + fmt.Fprintln(bw, "protocol\tsrc\tsport\tdst\tdport\texpected") + for i := 0; i < *flows; i++ { + f := s.Next() + fmt.Fprintf(bw, "%d\t%s\t%d\t%s\t%d\t%v\n", f.Protocol, f.SrcIP, f.SrcPort, f.DstIP, f.DstPort, fx.Expect(dir, f)) + } + return + } + + if err := fx.WriteYAML(bw, policyscale.ResourceOptions{Selector: *selector}); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/hack/deps.txt b/hack/deps.txt index c9f25f3e0d9..60cb491acd8 100644 --- a/hack/deps.txt +++ b/hack/deps.txt @@ -111,14 +111,20 @@ sigs.k8s.io/yaml v1.6.0 local:api/pkg/apis/projectcalico/v3 local:api/pkg/defaults local:api/pkg/lib/numorstring +local:app-policy/policyscale +local:app-policy/policystore +local:app-policy/types local:crypto/pkg/tls local:felix/ip +local:felix/proto +local:felix/types local:hack/cmd/calico-selector local:hack/cmd/coalesce-imports local:hack/cmd/deps local:hack/cmd/format-go-file local:hack/cmd/gomodder local:hack/cmd/ipam-hammer +local:hack/cmd/policyscale local:hack/perf/cmd/send-perf-results local:hack/test/spider local:lib/logrusr