diff --git a/ROADMAP.md b/ROADMAP.md index df48707..ce83015 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -170,9 +170,10 @@ Removed from the old "already correct" list — disproven by this review: ### U5. LLM format quality [S] ✅ done - Shipped 2026-07-15: deterministic ordering (category ties break lexicographically; conclusion max-pick tie-broken — both pinned by run-50-times tests); new `## Data Caveats` section narrates diagnostics and `## Glossary` defines volatility/persistence/q-value/billable semantics; volatile-step index keyed by (workflow, job); `[COST]` priorities gate on PriorityScore ≥ 50 like the suggestions. -### U6. Exit-code semantics for CI gating [M] -- Exits 0/1 only (`cmd/ci-snitch/main.go:36-40`). A `--fail-on regression|failure-rate>N` mode makes ci-snitch usable as a CI gate; pairs with F7. -- **Files:** `cmd/ci-snitch/`, docs +### U6. Exit-code semantics for CI gating [M] ✅ done +- Shipped 2026-07-15: `--fail-on regression,failure-rate>N` exits 2 (vs 1 = operational error) with one `fail-on:` reason per tripped finding on stderr, printed even in quiet mode; conditions validated before any fetch. Document in README via H10. + +**U section complete.** ### U7. Output polish batch [S total] ✅ done - Shipped 2026-07-15: hours render as "2h30m"; `|` escaped in markdown/LLM table names; JSON emits `"diagnostics": []`/`"findings": []` instead of null (fixed at the formatter boundary); `Diagnostic.String()` shows the wrapped cause; percentile displays floor (no more "slower than 100% of runs"); `-q/--quiet` silences stderr entirely; `--format` shell completion; `--since 0d`/future dates rejected before any API call; `compactResult` comment matches the code. diff --git a/cmd/ci-snitch/analyze.go b/cmd/ci-snitch/analyze.go index dab7e72..65a9773 100644 --- a/cmd/ci-snitch/analyze.go +++ b/cmd/ci-snitch/analyze.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cobra" + "github.com/vertti/ci-snitch/internal/analyze" "github.com/vertti/ci-snitch/internal/app" "github.com/vertti/ci-snitch/internal/github" "github.com/vertti/ci-snitch/internal/output" @@ -30,6 +31,7 @@ func newAnalyzeCmd() *cobra.Command { includeFailures bool verbose bool quiet bool + failOn string ) cmd := &cobra.Command{ @@ -49,6 +51,10 @@ If no repository is specified, detects the GitHub remote from the current direct if rawOutput != "" && format != "llm" { return errors.New("--raw-output requires --format llm") } + failConds, err := parseFailOn(failOn) + if err != nil { + return err + } var repo string if len(args) > 0 { @@ -138,7 +144,11 @@ If no repository is specified, detects the GitHub remote from the current direct prog.Log("Format: %s", time.Since(formatStart)) } prog.Log("Total: %s", time.Since(totalStart)) - return err + if err != nil { + return err + } + + return applyFailOnGate(failConds, &result) }, } @@ -151,6 +161,7 @@ If no repository is specified, detects the GitHub remote from the current direct cmd.Flags().BoolVar(&includeFailures, "include-failures", false, "include failed runs in analysis") cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "verbose output (show fetch details)") cmd.Flags().BoolVarP(&quiet, "quiet", "q", false, "suppress progress and diagnostic output on stderr") + cmd.Flags().StringVar(&failOn, "fail-on", "", "exit 2 when conditions match: regression, failure-rate>N (comma-separated)") _ = cmd.RegisterFlagCompletionFunc("format", cobra.FixedCompletions( []string{"table", "json", "markdown", "llm"}, cobra.ShellCompDirectiveNoFileComp)) @@ -213,3 +224,17 @@ func validateSincePast(t, now time.Time) error { } return nil } + +// applyFailOnGate prints tripped-condition reasons to stderr (even in quiet +// mode — the whole point of --fail-on is telling CI why the build failed) and +// returns exit code 2 via exitCodeError. +func applyFailOnGate(conds []failOnCondition, result *analyze.AnalysisResult) error { + reasons := evaluateFailOn(conds, result) + if len(reasons) == 0 { + return nil + } + for _, r := range reasons { + _, _ = fmt.Fprintf(os.Stderr, "fail-on: %s\n", r) + } + return &exitCodeError{code: 2, msg: fmt.Sprintf("%d --fail-on condition(s) tripped", len(reasons))} +} diff --git a/cmd/ci-snitch/failon.go b/cmd/ci-snitch/failon.go new file mode 100644 index 0000000..4c4e0e4 --- /dev/null +++ b/cmd/ci-snitch/failon.go @@ -0,0 +1,80 @@ +package main + +import ( + "fmt" + "strconv" + "strings" + + "github.com/vertti/ci-snitch/internal/analyze" +) + +// exitCodeError carries a specific process exit code through cobra's error +// path: 2 = a --fail-on gate tripped (data-driven), 1 = operational error. +type exitCodeError struct { + code int + msg string +} + +func (e *exitCodeError) Error() string { return e.msg } +func (e *exitCodeError) Code() int { return e.code } + +const ( + failOnRegression = "regression" + failOnFailureRate = "failure-rate" +) + +// failOnCondition is one parsed --fail-on condition. +type failOnCondition struct { + kind string // failOnRegression or failOnFailureRate + threshold float64 // percent, for failure-rate +} + +// parseFailOn parses a comma-separated --fail-on spec: +// "regression", "failure-rate>N", or both. +func parseFailOn(spec string) ([]failOnCondition, error) { + if spec == "" { + return nil, nil + } + var conds []failOnCondition + for part := range strings.SplitSeq(spec, ",") { + part = strings.TrimSpace(part) + switch { + case part == failOnRegression: + conds = append(conds, failOnCondition{kind: failOnRegression}) + case strings.HasPrefix(part, "failure-rate>"): + raw := strings.TrimPrefix(part, "failure-rate>") + threshold, err := strconv.ParseFloat(raw, 64) + if err != nil || threshold < 0 || threshold >= 100 { + return nil, fmt.Errorf("--fail-on failure-rate threshold must be a percentage in [0, 100), got %q", raw) + } + conds = append(conds, failOnCondition{kind: failOnFailureRate, threshold: threshold}) + default: + return nil, fmt.Errorf("unknown --fail-on condition %q (supported: regression, failure-rate>N)", part) + } + } + return conds, nil +} + +// evaluateFailOn returns one reason string per finding that trips a condition. +func evaluateFailOn(conds []failOnCondition, result *analyze.AnalysisResult) []string { + var reasons []string + for _, cond := range conds { + for _, f := range result.Findings { + switch cond.kind { + case failOnRegression: + d, ok := f.Detail.(analyze.ChangePointDetail) + if ok && d.Category == analyze.CategoryRegression { + reasons = append(reasons, fmt.Sprintf("regression: %s / %s %+.0f%%", + d.WorkflowName, d.JobName, d.PctChange)) + } + case failOnFailureRate: + d, ok := f.Detail.(analyze.FailureDetail) + if ok && d.FailureRate*100 > cond.threshold { + reasons = append(reasons, fmt.Sprintf("failure rate: %s at %.0f%% (threshold %.0f%%)", + d.Workflow, d.FailureRate*100, cond.threshold)) + } + } + } + } + return reasons +} diff --git a/cmd/ci-snitch/failon_test.go b/cmd/ci-snitch/failon_test.go new file mode 100644 index 0000000..95c86e7 --- /dev/null +++ b/cmd/ci-snitch/failon_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/vertti/ci-snitch/internal/analyze" +) + +func TestParseFailOn(t *testing.T) { + tests := []struct { + input string + wantErr string + }{ + {input: "regression"}, + {input: "failure-rate>25"}, + {input: "regression,failure-rate>10"}, + {input: "bogus", wantErr: "unknown --fail-on condition"}, + {input: "failure-rate>abc", wantErr: "threshold"}, + {input: "failure-rate>-5", wantErr: "threshold"}, + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + _, err := parseFailOn(tt.input) + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + require.NoError(t, err) + }) + } +} + +func TestFailOnEvaluate(t *testing.T) { + result := &analyze.AnalysisResult{Findings: []analyze.Finding{ + {Type: analyze.TypeChangepoint, Detail: analyze.ChangePointDetail{ + WorkflowName: "CI", JobName: "build", Category: analyze.CategoryRegression, PctChange: 40, + }}, + {Type: analyze.TypeChangepoint, Detail: analyze.ChangePointDetail{ + WorkflowName: "CI", JobName: "lint", Category: analyze.CategorySpeedup, PctChange: -20, + }}, + {Type: analyze.TypeFailure, Detail: analyze.FailureDetail{ + Workflow: "CI", FailureRate: 0.15, + }}, + }} + + conds, err := parseFailOn("regression") + require.NoError(t, err) + reasons := evaluateFailOn(conds, result) + require.Len(t, reasons, 1, "one regression must trip the gate") + assert.Contains(t, reasons[0], "build") + + conds, err = parseFailOn("failure-rate>10") + require.NoError(t, err) + reasons = evaluateFailOn(conds, result) + require.Len(t, reasons, 1, "15% > 10% must trip") + assert.Contains(t, reasons[0], "15%") + + conds, err = parseFailOn("failure-rate>20") + require.NoError(t, err) + assert.Empty(t, evaluateFailOn(conds, result), "15% is under a 20% threshold") + + // A speedup is not a regression. + noRegression := &analyze.AnalysisResult{Findings: result.Findings[1:]} + conds, _ = parseFailOn("regression") + assert.Empty(t, evaluateFailOn(conds, noRegression)) +} + +func TestExitCodeError(t *testing.T) { + err := &exitCodeError{code: 2, msg: "gate tripped"} + assert.Equal(t, 2, err.Code()) + assert.Equal(t, "gate tripped", err.Error()) +} diff --git a/cmd/ci-snitch/main.go b/cmd/ci-snitch/main.go index 301e143..9fc3640 100644 --- a/cmd/ci-snitch/main.go +++ b/cmd/ci-snitch/main.go @@ -3,6 +3,7 @@ package main import ( "context" + "errors" "fmt" "os" "os/signal" @@ -44,6 +45,12 @@ func main() { err := newRootCmd().ExecuteContext(ctx) stop() if err != nil { + // --fail-on gates exit 2 so CI can distinguish "findings tripped the + // gate" from operational failure (1). + var ec *exitCodeError + if errors.As(err, &ec) { + os.Exit(ec.Code()) + } os.Exit(1) } }