Refactor offline checks

cmd/pint/main.go

@@ -70,7 +70,7 @@ func newApp() *cli.Command {
 				Name:    offlineFlag,
 				Aliases: []string{"o"},
 				Value:   false,
-				Usage:   "Disable all checks that send live queries to Prometheus servers.",
+				Usage:   "Offline mode, no network requests will be sent, some parts of check logic might be disabled or checks might be skipped entirely.",
 			},
 			&cli.BoolFlag{
 				Name:    showDupsFlag,
@@ -123,10 +123,7 @@ func actionSetup(c *cli.Command) (meta actionMeta, err error) {
 		meta.cfg.Checks.Enabled = enabled
 	}
 
-	if c.Bool(offlineFlag) {
-		meta.isOffline = true
-		meta.cfg.DisableOnlineChecks()
-	}
+	meta.isOffline = c.Bool(offlineFlag)
 
 	return meta, nil
 }

cmd/pint/scan.go

@@ -43,13 +43,14 @@ func checkRules(ctx context.Context, workers int, isOffline bool, gen *config.Pr
 	var reports []reporter.Report
 
 	ctx = context.WithValue(ctx, promapi.AllPrometheusServers, gen.Servers())
+	ctx = promapi.WithOffline(ctx, isOffline)
 	for _, s := range cfg.Check {
 		settings, _ := s.Decode()
 		key := checks.SettingsKey(s.Name)
 		ctx = context.WithValue(ctx, key, settings)
 	}
 
-	var onlineChecksCount, offlineChecksCount, checkedEntriesCount atomic.Int64
+	var checkedEntriesCount, totalChecksCount atomic.Int64
 	for _, entry := range entries {
 		switch {
 		case entry.PathError != nil && entry.State == discovery.Removed:
@@ -91,11 +92,7 @@ func checkRules(ctx context.Context, workers int, isOffline bool, gen *config.Pr
 			checkList := cfg.GetChecksForEntry(ctx, gen, entry)
 			for _, check := range checkList {
 				checkIterationChecks.Inc()
-				if check.Meta().Online {
-					onlineChecksCount.Add(1)
-				} else {
-					offlineChecksCount.Add(1)
-				}
+				totalChecksCount.Add(1)
 				concurrencyLimit <- struct{}{}
 				wg.Go(func() {
 					defer func() { <-concurrencyLimit }()
@@ -117,8 +114,7 @@ func checkRules(ctx context.Context, workers int, isOffline bool, gen *config.Pr
 	summary.Duration = time.Since(start)
 	summary.TotalEntries = len(entries)
 	summary.CheckedEntries = checkedEntriesCount.Load()
-	summary.OnlineChecks = onlineChecksCount.Load()
-	summary.OfflineChecks = offlineChecksCount.Load()
+	summary.TotalChecks = totalChecksCount.Load()
 
 	for _, prom := range gen.Servers() {
 		for api, names := range prom.GetDisabledChecks() {

cmd/pint/scan_test.go

@@ -18,7 +18,7 @@ type testCheck struct {
 }
 
 func (tc testCheck) Meta() checks.CheckMeta {
-	return checks.CheckMeta{Online: false}
+	return checks.CheckMeta{}
 }
 
 func (tc testCheck) String() string {

cmd/pint/tests/0032_ci_github.txt

@@ -218,8 +218,7 @@ body: |+
     | Number of rules parsed | 3 |
     | Number of rules checked | 3 |
     | Number of problems found | 3 |
-    | Number of offline checks | 21 |
-    | Number of online checks | 0 |
+    | Number of checks | 21 |
     | Checks duration | 0 |
 
     </p>

cmd/pint/tests/0033_ci_github_multi.txt

@@ -206,8 +206,7 @@ body: |+
     | Number of rules parsed | 2 |
     | Number of rules checked | 2 |
     | Number of problems found | 4 |
-    | Number of offline checks | 20 |
-    | Number of online checks | 0 |
+    | Number of checks | 20 |
     | Checks duration | 0 |
 
     </p>

cmd/pint/tests/0054_watch_metrics_prometheus.txt

@@ -102,16 +102,16 @@ pint_last_run_duration_seconds
 pint_last_run_time_seconds
 # HELP pint_problem Prometheus rule problem reported by pint.
 # TYPE pint_problem gauge
-pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",owner="",problem="unable to run checks: Couldn't run some online checks due to `prom1` Prometheus server at http://127.0.0.1:7054 error: `bad_data: bogus query`.",reporter="promql/series",severity="bug"}
-pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",owner="",problem="unable to run checks: Couldn't run some online checks due to `prom1` Prometheus server at http://127.0.0.1:7054 error: `server_error: 500 Internal Server Error`.",reporter="promql/rate",severity="bug"}
-pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",owner="",problem="unable to run checks: Couldn't run some online checks due to `prom2` Prometheus server at http://127.0.0.1:1054 error: `connection refused`.",reporter="promql/counter",severity="bug"}
-pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",owner="",problem="unable to run checks: Couldn't run some online checks due to `prom2` Prometheus server at http://127.0.0.1:1054 error: `connection refused`.",reporter="promql/offset",severity="bug"}
-pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",owner="",problem="unable to run checks: Couldn't run some online checks due to `prom2` Prometheus server at http://127.0.0.1:1054 error: `connection refused`.",reporter="promql/range_query",severity="bug"}
-pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",owner="",problem="unable to run checks: Couldn't run some online checks due to `prom2` Prometheus server at http://127.0.0.1:1054 error: `connection refused`.",reporter="promql/rate",severity="bug"}
-pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",owner="",problem="unable to run checks: Couldn't run some online checks due to `prom2` Prometheus server at http://127.0.0.1:1054 error: `connection refused`.",reporter="promql/series",severity="bug"}
+pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",owner="",problem="unable to run checks: Couldn't run some checks due to `prom1` Prometheus server at http://127.0.0.1:7054 error: `bad_data: bogus query`.",reporter="promql/series",severity="bug"}
+pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",owner="",problem="unable to run checks: Couldn't run some checks due to `prom1` Prometheus server at http://127.0.0.1:7054 error: `server_error: 500 Internal Server Error`.",reporter="promql/rate",severity="bug"}
+pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",owner="",problem="unable to run checks: Couldn't run some checks due to `prom2` Prometheus server at http://127.0.0.1:1054 error: `connection refused`.",reporter="promql/counter",severity="bug"}
+pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",owner="",problem="unable to run checks: Couldn't run some checks due to `prom2` Prometheus server at http://127.0.0.1:1054 error: `connection refused`.",reporter="promql/offset",severity="bug"}
+pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",owner="",problem="unable to run checks: Couldn't run some checks due to `prom2` Prometheus server at http://127.0.0.1:1054 error: `connection refused`.",reporter="promql/range_query",severity="bug"}
+pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",owner="",problem="unable to run checks: Couldn't run some checks due to `prom2` Prometheus server at http://127.0.0.1:1054 error: `connection refused`.",reporter="promql/rate",severity="bug"}
+pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",owner="",problem="unable to run checks: Couldn't run some checks due to `prom2` Prometheus server at http://127.0.0.1:1054 error: `connection refused`.",reporter="promql/series",severity="bug"}
 pint_problem{filename="rules/1.yml",kind="recording",name="broken",owner="",problem="PromQL syntax error: unexpected right parenthesis ')'",reporter="promql/syntax",severity="fatal"}
-pint_problem{filename="rules/2.yml",kind="alerting",name="comparison",owner="bob and alice",problem="unable to run checks: Couldn't run some online checks due to `prom1` Prometheus server at http://127.0.0.1:7054 error: `server_error: 500 Internal Server Error`.",reporter="alerts/external_labels",severity="bug"}
-pint_problem{filename="rules/2.yml",kind="alerting",name="comparison",owner="bob and alice",problem="unable to run checks: Couldn't run some online checks due to `prom2` Prometheus server at http://127.0.0.1:1054 error: `connection refused`.",reporter="alerts/external_labels",severity="bug"}
+pint_problem{filename="rules/2.yml",kind="alerting",name="comparison",owner="bob and alice",problem="unable to run checks: Couldn't run some checks due to `prom1` Prometheus server at http://127.0.0.1:7054 error: `server_error: 500 Internal Server Error`.",reporter="alerts/external_labels",severity="bug"}
+pint_problem{filename="rules/2.yml",kind="alerting",name="comparison",owner="bob and alice",problem="unable to run checks: Couldn't run some checks due to `prom2` Prometheus server at http://127.0.0.1:1054 error: `connection refused`.",reporter="alerts/external_labels",severity="bug"}
 # HELP pint_problems Total number of problems reported by pint.
 # TYPE pint_problems gauge
 pint_problems

cmd/pint/tests/0057_watch_metrics_prometheus_ignore.txt

@@ -100,7 +100,7 @@ pint_last_run_duration_seconds
 pint_last_run_time_seconds
 # HELP pint_problem Prometheus rule problem reported by pint.
 # TYPE pint_problem gauge
-pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",owner="",problem="unable to run checks: Couldn't run some online checks due to `prom1` Prometheus server at http://127.0.0.1:7057 error: `bad_data: bogus query`.",reporter="promql/series",severity="bug"}
+pint_problem{filename="rules/1.yml",kind="recording",name="aggregate",owner="",problem="unable to run checks: Couldn't run some checks due to `prom1` Prometheus server at http://127.0.0.1:7057 error: `bad_data: bogus query`.",reporter="promql/series",severity="bug"}
 pint_problem{filename="rules/1.yml",kind="recording",name="broken",owner="",problem="PromQL syntax error: unexpected right parenthesis ')'",reporter="promql/syntax",severity="fatal"}
 # HELP pint_problems Total number of problems reported by pint.
 # TYPE pint_problems gauge

cmd/pint/tests/0083_github_action.txt

@@ -174,8 +174,7 @@ body: |+
     | Number of rules parsed | 2 |
     | Number of rules checked | 2 |
     | Number of problems found | 2 |
-    | Number of offline checks | 10 |
-    | Number of online checks | 0 |
+    | Number of checks | 10 |
     | Checks duration | 0 |
 
     </p>

cmd/pint/tests/0084_github_action_override.txt

@@ -177,8 +177,7 @@ body: |+
     | Number of rules parsed | 2 |
     | Number of rules checked | 2 |
     | Number of problems found | 2 |
-    | Number of offline checks | 10 |
-    | Number of online checks | 0 |
+    | Number of checks | 10 |
     | Checks duration | 0 |
 
     </p>

cmd/pint/tests/0085_github_no_envs.txt

@@ -172,8 +172,7 @@ body: |+
     | Number of rules parsed | 2 |
     | Number of rules checked | 2 |
     | Number of problems found | 2 |
-    | Number of offline checks | 10 |
-    | Number of online checks | 0 |
+    | Number of checks | 10 |
     | Checks duration | 0 |
 
     </p>

cmd/pint/tests/0098_rule_file_symlink_gh.txt

@@ -151,8 +151,7 @@ body: |+
     | Number of rules parsed | 4 |
     | Number of rules checked | 4 |
     | Number of problems found | 2 |
-    | Number of offline checks | 40 |
-    | Number of online checks | 0 |
+    | Number of checks | 40 |
     | Checks duration | 0 |
 
     </p>

cmd/pint/tests/0103_file_disable.txt

@@ -15,7 +15,7 @@ level=DEBUG msg="Generated all Prometheus servers" count=1
 level=DEBUG msg="Found recording rule" path=rules/0001.yml record=colo:test1 lines=10-11 state=noop
 level=DEBUG msg="Configured checks for rule" enabled=["promql/syntax","alerts/for","alerts/comparison","alerts/template","promql/fragile","promql/regexp","rule/dependency","promql/impossible","promql/nan","group/interval","promql/vector_matching(prom)","labels/conflict(prom)","alerts/external_labels(prom)","alerts/absent(prom)","promql/features(prom)"] path=rules/0001.yml rule=colo:test1
 -- rules/0001.yml --
-# This should skip all online checks
+# This should skip all checks that require Prometheus API
 # pint file/disable promql/series
 #  pint  file/disable  promql/rate
 #

cmd/pint/tests/0104_file_ignore_prom.txt

@@ -9,7 +9,7 @@ stderr 'level=ERROR msg="Query returned an error" err="502 Bad Gateway" uri=http
 stderr 'level=ERROR msg="Query returned an error" err="502 Bad Gateway" uri=http://127.0.0.1:7104 query=count\(\\nfoo\\n\)'
 stderr 'level=INFO msg="Problems found" Bug=4'
 -- rules/0001.yml --
-# This should skip all online checks
+# This should skip all checks that require Prometheus API
 # pint file/disable promql/series
 #  pint  file/disable  promql/rate
 #

cmd/pint/tests/0108_rule_duplicate.txt

@@ -21,8 +21,8 @@ Bug: unable to run checks (labels/conflict)
   ---> rules/0001.yml:5-8 -> `colo:labels:mismatch` [+4 duplicates]
 5 | - record: "colo:labels:mismatch"
                ^^^^^^^^^^^^^^^^^^^^
-               Couldn't run some online checks due to `prom` Prometheus server at http://127.0.0.1:7108
-               error: `connection refused`.
+               Couldn't run some checks due to `prom` Prometheus server at http://127.0.0.1:7108 error:
+               `connection refused`.
 
 Bug: duplicated recording rule (rule/duplicate)
   ---> rules/0001.yml:9-12 -> `colo:labels:equal`

cmd/pint/tests/0115_file_disable_tag.txt

@@ -21,8 +21,8 @@ Warning: unable to run checks (promql/counter)
   ---> rules/0001.yml:7-9 -> `colo:test1`
 7 | - record: "colo:test1"
                ^^^^^^^^^^
-               Couldn't run some online checks due to `prom` Prometheus server at http://127.0.0.1:7103
-               error: `connection refused`.
+               Couldn't run some checks due to `prom` Prometheus server at http://127.0.0.1:7103 error:
+               `connection refused`.
 
 level=INFO msg="Problems found" Warning=1
 -- rules/0001.yml --

cmd/pint/tests/0118_ci_dir_move.txt

@@ -93,8 +93,7 @@ body: |+
     | Number of rules parsed | 2 |
     | Number of rules checked | 2 |
     | Number of problems found | 0 |
-    | Number of offline checks | 0 |
-    | Number of online checks | 0 |
+    | Number of checks | 0 |
     | Checks duration | 0 |
 
     </p>

cmd/pint/tests/0119_ci_fail_on_warning.txt

@@ -211,8 +211,7 @@ body: |+
     | Number of rules parsed | 2 |
     | Number of rules checked | 2 |
     | Number of problems found | 3 |
-    | Number of offline checks | 11 |
-    | Number of online checks | 0 |
+    | Number of checks | 11 |
     | Checks duration | 0 |
 
     </p>

cmd/pint/tests/0246_ci_github_new_file.txt

@@ -123,8 +123,7 @@ body: |+
     | Number of rules parsed | 1 |
     | Number of rules checked | 1 |
     | Number of problems found | 1 |
-    | Number of offline checks | 10 |
-    | Number of online checks | 0 |
+    | Number of checks | 10 |
     | Checks duration | 0 |
 
     </p>

cmd/pint/tests/0247_ci_github_rename.txt

@@ -164,8 +164,7 @@ body: |+
     | Number of rules parsed | 1 |
     | Number of rules checked | 1 |
     | Number of problems found | 2 |
-    | Number of offline checks | 9 |
-    | Number of online checks | 0 |
+    | Number of checks | 9 |
     | Checks duration | 0 |
 
     </p>

cmd/pint/tests/0248_ci_github_deleted_line.txt

@@ -169,8 +169,7 @@ body: |+
     | Number of rules parsed | 5 |
     | Number of rules checked | 5 |
     | Number of problems found | 1 |
-    | Number of offline checks | 2 |
-    | Number of online checks | 0 |
+    | Number of checks | 2 |
     | Checks duration | 0 |
 
     </p>

cmd/pint/tests/0249_ci_github_shifted_line.txt

@@ -165,8 +165,7 @@ body: |+
     | Number of rules parsed | 2 |
     | Number of rules checked | 2 |
     | Number of problems found | 2 |
-    | Number of offline checks | 20 |
-    | Number of online checks | 0 |
+    | Number of checks | 20 |
     | Checks duration | 0 |
 
     </p>

cmd/pint/tests/0251_ci_github_line_outside_diff.txt

@@ -152,8 +152,7 @@ body: |+
     | Number of rules parsed | 1 |
     | Number of rules checked | 1 |
     | Number of problems found | 1 |
-    | Number of offline checks | 10 |
-    | Number of online checks | 0 |
+    | Number of checks | 10 |
     | Checks duration | 0 |
 
     </p>

cmd/pint/tests/0270_ci_github_long_expr.txt

@@ -238,8 +238,7 @@ body: |+
     | Number of rules parsed | 2 |
     | Number of rules checked | 2 |
     | Number of problems found | 3 |
-    | Number of offline checks | 20 |
-    | Number of online checks | 0 |
+    | Number of checks | 20 |
     | Checks duration | 0 |
 
     </p>

cmd/pint/tests/0271_ci_github_stale_general_comment.txt

@@ -104,8 +104,7 @@ body: |+
     | Number of rules parsed | 1 |
     | Number of rules checked | 1 |
     | Number of problems found | 0 |
-    | Number of offline checks | 10 |
-    | Number of online checks | 0 |
+    | Number of checks | 10 |
     | Checks duration | 0 |
 
     </p>

docs/index.md

@@ -13,13 +13,13 @@ pint is a Prometheus rule linter/validator.
 
 pint will run checks on Prometheus alerting & recording rules to detect potential problems
 with those rules.
-Some checks rely only on the rule itself and can be run "offline" - without talking to any
-Prometheus server.
-You can run pint in "offline" mode if you:
-
-- Don't pass any configuration file to pint.
-- You pass configuration file to pint that **doesn't** contain any `prometheus` definition.
-- You pass `--offline` flag to `pint` command.
+You can run pint in "offline" mode by passing the `--offline` flag.
+In this mode no network requests will be sent. Some checks might be skipped
+entirely while others will run with reduced functionality — only the parts
+of their logic that don't require network access will be executed.
+This means that passing `--offline` will disable Prometheus API queries and so
+any check, or part of a check, that needs to talk to a live Prometheus server
+will be skipped.
 
 Most checks included in pint will require sending queries to a running Prometheus server where
 those rules are, or would be, deployed.

internal/checks/alerts_absent.go

@@ -40,7 +40,6 @@ func (c AlertsAbsentCheck) Meta() CheckMeta {
 			discovery.Modified,
 			discovery.Moved,
 		},
-		Online:        true,
 		AlwaysEnabled: false,
 	}
 }
@@ -54,6 +53,10 @@ func (c AlertsAbsentCheck) Reporter() string {
 }
 
 func (c AlertsAbsentCheck) Check(ctx context.Context, entry *discovery.Entry, _ []*discovery.Entry) (problems []Problem) {
+	if promapi.IsOffline(ctx) {
+		return problems
+	}
+
 	if entry.Rule.AlertingRule == nil {
 		return problems
 	}

internal/checks/alerts_absent_test.go

@@ -1,6 +1,7 @@
 package checks_test
 
 import (
+	"context"
 	"net/http"
 	"testing"
 	"time"
@@ -15,6 +16,15 @@ func newAlertsAbsentCheck(prom *promapi.FailoverGroup) checks.RuleChecker {
 
 func TestAlertsAbsentCheck(t *testing.T) {
 	testCases := []checkTest{
+		{
+			description: "offline",
+			content:     "- alert: foo\n  expr: absent(foo)\n",
+			checker:     newAlertsAbsentCheck,
+			prometheus:  newSimpleProm,
+			ctx: func(ctx context.Context, _ string) context.Context {
+				return promapi.WithOffline(ctx, true)
+			},
+		},
 		{
 			description: "ignores recording rules",
 			content:     "- record: foo\n  expr: sum(foo)\n",