✨ Workflow models: ingest issue templates, triage, owners and branch rules into drafts

changelog.d/added-workflowmodel-ingest-intake.md

@@ -0,0 +1 @@
+- **workflow models:** ingesting a repository now proposes its *intake half* alongside its pipelines: issue templates become `requested-<slug>` intake states and events, a triage label (from templates, `settings.yml`, `labels.yml`, `labeler.yml` or `CONTRIBUTING.md`) becomes a `triage` state and gate, PR-template checklist items become manual checks, `CODEOWNERS`/Prow `OWNERS` become `approver`/`reviewer` roles owning the review gate, a `MAINTAINERS` file's `@handles` become the `maintainer` role's members, `kind/*`/`priority/*`/`area/*` labels are carried as classification rows, and the default branch's protection or rulesets (fetched when ingesting by `repo`, or a probot `settings.yml`) become required CI checks, `review.minApprovals`, a `codeowner-review` gate and bookkeeping notes. Everything is `inferred` and cites `path:line` or the API endpoint; ingesting `cncf/techdocs` now proposes its triage state. Trees without these inputs produce the byte-identical model as before; `workflow-survey` counts the proposals per repository ([#17620](https://github.com/hivecommons/hive/issues/17620))

src/cmd/hive/workflow_ingest.go

@@ -2,7 +2,9 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
+	"log"
 
 	"github.com/hivecommons/hive/pkg/github"
 	"github.com/hivecommons/hive/pkg/workflowmodel/ingest"
@@ -14,9 +16,11 @@ var errNoForgeClient = errors.New("no GitHub client")
 
 // workflowRepoFiles is the dashboard's WorkflowRepoFiles dependency
 // (POST /api/workflows/ingest, RFC #17620 P3): it downloads the repo's
-// default branch read-only, keeps only the CI definition files and
-// removes the checkout. client is read per call so a credential rebuild is
-// followed.
+// default branch read-only, keeps only the CI definition and process
+// convention files and removes the checkout. The default branch's rules
+// (protection or rulesets, P5) ride along at ingest.BranchRulesPath when
+// the forge reports any; a failure to read them is logged, not fatal.
+// client is read per call so a credential rebuild is followed.
 func workflowRepoFiles(client func() *github.Client) func(ctx context.Context, repo string) (map[string][]byte, string, error) {
 	return func(ctx context.Context, repo string) (map[string][]byte, string, error) {
 		c := client()
@@ -32,6 +36,14 @@ func workflowRepoFiles(client func() *github.Client) func(ctx context.Context, r
 		if err != nil {
 			return nil, "", err
 		}
+		switch rules, err := c.FetchBranchRules(ctx, repo, co.Ref); {
+		case err != nil:
+			log.Printf("workflow ingest: %s branch rules: %v", repo, err)
+		case rules != nil:
+			if b, err := json.Marshal(rules); err == nil {
+				files[ingest.BranchRulesPath] = b
+			}
+		}
 		return files, co.Ref, nil
 	}
 }

src/cmd/workflow-survey/main.go

@@ -74,6 +74,21 @@ type repoResult struct {
 	CompileError  string         `json:"compileError,omitempty"`
 	CompileCodes  map[string]int `json:"compileCodes,omitempty"`
 	LifecycleCols int            `json:"lifecycleColumns"`
+	// Intake is what the P5 intake half proposed from the repository's
+	// process conventions (issue/PR templates, OWNERS, branch rules).
+	Intake intakeStats `json:"intake"`
+}
+
+// intakeStats counts the intake-half proposals in one lifecycle model.
+type intakeStats struct {
+	IntakeStates   int  `json:"intakeStates"`
+	Triage         bool `json:"triage"`
+	MemberRoles    int  `json:"memberRoles"`
+	InferredGates  int  `json:"inferredGates"`
+	ManualChecks   int  `json:"manualChecks"`
+	CIChecks       int  `json:"ciChecks"`
+	BranchRules    bool `json:"branchRules"`
+	IntakeFindings int  `json:"intakeFindings"`
 }
 
 type summary struct {
@@ -92,10 +107,22 @@ type summary struct {
 	Severities       map[workflowmodel.Severity]int `json:"severities"`
 	ColumnSets       map[string]int                 `json:"columnSets"`
 	ReposChangedCols int                            `json:"reposWithChangedColumns"`
-	ByProject        map[string]*projectStats       `json:"byProject"`
-	Worst            []repoResult                   `json:"worst"`
-	Top              []repoResult                   `json:"top"`
-	Repos_           []repoResult                   `json:"results"`
+	// Intake counts repos where the intake half proposed each element.
+	Intake    intakeSummary            `json:"intake"`
+	ByProject map[string]*projectStats `json:"byProject"`
+	Worst     []repoResult             `json:"worst"`
+	Top       []repoResult             `json:"top"`
+	Repos_    []repoResult             `json:"results"`
+}
+
+// intakeSummary counts repositories, not elements.
+type intakeSummary struct {
+	ReposWithIntakeStates int `json:"reposWithIntakeStates"`
+	ReposWithTriage       int `json:"reposWithTriage"`
+	ReposWithRoles        int `json:"reposWithRoles"`
+	ReposWithGates        int `json:"reposWithGates"`
+	ReposWithChecks       int `json:"reposWithChecks"`
+	ReposWithBranchRules  int `json:"reposWithBranchRules"`
 }
 
 type projectStats struct {
@@ -272,6 +299,7 @@ func surveyRepo(dir, repo, outDir string) (repoResult, error) {
 	} else {
 		res.Compiled = true
 	}
+	res.Intake = countIntake(lm)
 
 	if err := os.MkdirAll(outDir, 0o755); err != nil {
 		return res, err
@@ -296,6 +324,50 @@ func surveyRepo(dir, repo, outDir string) (repoResult, error) {
 	return res, writeJSON(filepath.Join(outDir, "result.json"), res)
 }
 
+// countIntake tallies the intake-half proposals in a lifecycle model.
+func countIntake(m *workflowmodel.Model) intakeStats {
+	var st intakeStats
+	for _, s := range m.Spec.States {
+		switch {
+		case s.ID == ingest.StateTriage:
+			st.Triage = true
+		case strings.HasPrefix(s.ID, ingest.StateRequestedPrefix):
+			st.IntakeStates++
+		}
+	}
+	for _, r := range m.Spec.Roles {
+		if r.Kind == workflowmodel.RoleMembers {
+			st.MemberRoles++
+		}
+	}
+	for _, g := range m.Spec.Gates {
+		if g.Confidence == workflowmodel.ConfidenceInferred {
+			st.InferredGates++
+		}
+	}
+	for _, c := range m.Spec.Checks {
+		switch c.Kind {
+		case workflowmodel.CheckManual:
+			st.ManualChecks++
+		case workflowmodel.CheckCI:
+			st.CIChecks++
+			st.BranchRules = true
+		}
+	}
+	for _, g := range m.Spec.Gates {
+		if g.ID == ingest.GateCodeownerReview || (g.ID == ingest.GateReview && g.MinApprovals > 0) {
+			st.BranchRules = true
+		}
+	}
+	st.BranchRules = st.BranchRules || len(m.Spec.Bookkeeping) > 0
+	for _, f := range m.Spec.Coverage.Findings {
+		if strings.HasPrefix(f.Code, "issue-template-") {
+			st.IntakeFindings++
+		}
+	}
+	return st
+}
+
 func countConstructs(c map[string]int, p workflowmodel.Pipeline) {
 	c["pipelines"]++
 	if p.Concurrency != nil {
@@ -357,6 +429,24 @@ func accumulate(s *summary, r repoResult) {
 	if len(r.ColumnDiff) > 0 {
 		s.ReposChangedCols++
 	}
+	if r.Intake.IntakeStates > 0 {
+		s.Intake.ReposWithIntakeStates++
+	}
+	if r.Intake.Triage {
+		s.Intake.ReposWithTriage++
+	}
+	if r.Intake.MemberRoles > 0 {
+		s.Intake.ReposWithRoles++
+	}
+	if r.Intake.InferredGates > 0 {
+		s.Intake.ReposWithGates++
+	}
+	if r.Intake.ManualChecks+r.Intake.CIChecks > 0 {
+		s.Intake.ReposWithChecks++
+	}
+	if r.Intake.BranchRules {
+		s.Intake.ReposWithBranchRules++
+	}
 	for k, v := range r.Roles {
 		s.Roles[k] += v
 	}
@@ -411,6 +501,8 @@ func markdown(s *summary) string {
 		fmt.Fprintf(&b, "| %s | %d |\n", k, s.CompileCodes[k])
 	}
 	b.WriteString("\n")
+	fmt.Fprintf(&b, "## Intake proposals (RFC phase 5)\n\n| Proposed | Repos |\n|---|---|\n| Intake states (issue templates) | %d |\n| Triage state | %d |\n| Member roles (CODEOWNERS/OWNERS) | %d |\n| Inferred gates | %d |\n| Checks (PR template / required status) | %d |\n| Branch rules | %d |\n\n",
+		s.Intake.ReposWithIntakeStates, s.Intake.ReposWithTriage, s.Intake.ReposWithRoles, s.Intake.ReposWithGates, s.Intake.ReposWithChecks, s.Intake.ReposWithBranchRules)
 	b.WriteString("## Pipeline roles\n\n| Role | Pipelines |\n|---|---|\n")
 	for _, k := range sortedKeys(s.Roles) {
 		fmt.Fprintf(&b, "| %s | %d |\n", k, s.Roles[workflowmodel.Role(k)])

src/cmd/workflow-survey/main_test.go

@@ -44,6 +44,15 @@ func buildCorpus(t *testing.T) string {
 	if n == 0 {
 		t.Fatalf("no fixture repositories under %s", fixtureCorpus)
 	}
+	// One repo with process conventions but no pipelines exercises the
+	// intake counts (RFC phase 5).
+	intake, err := filepath.Abs(filepath.Join("..", "..", "pkg", "workflowmodel", "ingest", "testdata", "k8s-owners"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.CopyFS(filepath.Join(repos, "example__k8s-owners"), os.DirFS(intake)); err != nil {
+		t.Fatal(err)
+	}
 	// A stray file in repos/ must be ignored, as must a repo not in projects.json.
 	if err := os.WriteFile(filepath.Join(repos, "README.md"), []byte("x"), 0o644); err != nil {
 		t.Fatal(err)
@@ -82,8 +91,19 @@ func TestRealMainSurveysCorpus(t *testing.T) {
 	if err := json.Unmarshal(raw, &sum); err != nil {
 		t.Fatal(err)
 	}
-	if sum.Repos != 6 || sum.Valid != sum.Repos {
-		t.Errorf("repos=%d valid=%d, want 6 valid", sum.Repos, sum.Valid)
+	if sum.Repos != 7 || sum.Valid != sum.Repos {
+		t.Errorf("repos=%d valid=%d, want 7 valid", sum.Repos, sum.Valid)
+	}
+	if sum.Compiled != sum.Repos {
+		t.Errorf("compiled=%d of %d", sum.Compiled, sum.Repos)
+	}
+	if in := sum.Intake; in.ReposWithIntakeStates != 1 || in.ReposWithTriage != 1 || in.ReposWithRoles != 1 || in.ReposWithGates != 1 || in.ReposWithChecks != 1 || in.ReposWithBranchRules != 1 {
+		t.Errorf("intake summary %+v, want exactly the k8s-owners repo", in)
+	}
+	for _, r := range sum.Repos_ {
+		if r.Repo == "example/k8s-owners" && (r.Intake.IntakeStates != 3 || !r.Intake.Triage || r.Intake.MemberRoles != 2 || r.Intake.ManualChecks != 4 || r.Intake.CIChecks != 3 || r.Intake.IntakeFindings != 1) {
+			t.Errorf("k8s-owners intake %+v", r.Intake)
+		}
 	}
 	if sum.ReposWithGHA == 0 || sum.ReposWithTekton == 0 {
 		t.Errorf("expected both systems in the corpus: gha=%d tekton=%d", sum.ReposWithGHA, sum.ReposWithTekton)
@@ -94,7 +114,7 @@ func TestRealMainSurveysCorpus(t *testing.T) {
 	if got := sum.ByProject["graduated"]; got == nil || got.Repos != 1 {
 		t.Errorf("projects.json maturity join (case-insensitive) not applied: %+v", sum.ByProject)
 	}
-	if got := sum.ByProject["unknown"]; got == nil || got.Repos != 4 {
+	if got := sum.ByProject["unknown"]; got == nil || got.Repos != 5 {
 		t.Errorf("repos without a maturity should land in unknown: %+v", sum.ByProject)
 	}
 	if len(sum.Worst) != sum.Repos || len(sum.Top) != sum.Repos {
@@ -110,7 +130,7 @@ func TestRealMainSurveysCorpus(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	for _, want := range []string{"# Workflow model survey", "## Pipeline roles", "## Column sets (Activity Monitor)", "| graduated | 1 |", "trigger:pull_request"} {
+	for _, want := range []string{"# Workflow model survey", "## Pipeline roles", "## Column sets (Activity Monitor)", "## Intake proposals (RFC phase 5)", "| Triage state | 1 |", "| graduated | 1 |", "trigger:pull_request"} {
 		if !strings.Contains(string(md), want) {
 			t.Errorf("summary.md missing %q", want)
 		}

src/docs/design/workflow-models.md

@@ -331,3 +331,66 @@ exactly the pre-P4 path: one list, the built-in family, no model labels.
   phase 4 and follows the diagram editor.
 - A model without a `labels` block regenerates nothing (TechDocs as shipped); the preview still
   shows its commands and the built-in stages its numbered states land on.
+
+## Ingest: intake, triage, roles and branch rules (RFC phase 5)
+
+`ingest.Lifecycle` now reads the repository's process conventions next to its pipelines and adds
+an *intake half* to the draft: how work enters, who triages it, who approves a change and what the
+forge requires before merge. `ReadTree` keeps the files below in addition to the CI definitions
+(`ingest.IsIntakePath`); the dashboard seam (`WorkflowRepoFiles`) is unchanged. Everything this
+half adds is `confidence: inferred` and cites where it came from; the compiler warns once per
+element and a human confirms them in the editor exactly as for the pipeline half. Trees with none
+of these inputs produce the byte-identical model as before (`TestLifecyclePipelineOnlyUnchanged`).
+
+### Heuristics
+
+| Input | Read as | Proposes |
+|---|---|---|
+| `.github/ISSUE_TEMPLATE/*.{yml,yaml}` (issue forms), `*.md` and `.github/ISSUE_TEMPLATE.md` (front matter) | `name`, `labels` (list or comma string) | One `requested-<slug>` state per distinct label set in an **Intake** column after Open; event `issue-opened-with-template:<slug>` = `labels has <bind label>` (the template's first label that is not a triage label and not already bound); transitions open→requested, requested→triage (or →actionable when no triage label exists) and requested→closed; a `labels.entries` row per template label pointing at the state. |
+| `.github/ISSUE_TEMPLATE/config.yml` | `blank_issues_enabled` | Annotation `workflow.hivecommons.dev/blank-issues: disabled` when false (every issue arrives through a template). |
+| Labels seen in templates, `.github/settings.yml` (probot `labels:`), `.github/labels.yml` (sequence, `labels:` wrapper or mapping), `.github/labeler.yml` (top-level keys), `CONTRIBUTING.md` (backticked labels, or hyphen/slash labels at word boundaries) | Triage vocabulary: `needs-triage`/`triage/needed`/`status/needs-triage`/…; `triage/accepted`/`triage-accepted`/`accepted`; `triage/declined`/`wontfix`/`invalid`/`not-a-bug`; conventions `good first issue`, `help wanted` | A `triage` state in a **Triage** column after Intake; events `triage-requested` (`labels has <needs>`), `triage-accepted` (`labels has <accepted>`, or `labels lacks <needs>` when there is no accepted label), `triage-declined`; gate `triage-decision` released by `approver` when one exists, else `maintainer`, with reply `/label <accepted>`; transitions open→triage, triage→actionable (gated), triage→closed; label rows `needs → triage`, `accepted → actionable`, `declined → closed`; convention labels as plain rows. |
+| PR template (`.github/PULL_REQUEST_TEMPLATE.md`, `PULL_REQUEST_TEMPLATE.md`, `docs/…`, `.github/PULL_REQUEST_TEMPLATE/*.md`) | `- [ ]` / `* [ ]` / `[x]` items | `checks` entries `pr-template-<slug>` of kind `manual`, `source: path:line`. Duplicates by id are dropped. |
+| `CODEOWNERS` (root, `.github/`, `docs/`) | Every owner on every rule (users and `org/team`) | Role `approver` of kind `members` (params `members`, `source: path:line,…`). |
+| root `OWNERS`, `OWNERS_ALIASES` (Prow) | `approvers`, `reviewers`, `filters[".*"]`, aliases | Roles `approver` and `reviewer`; the review gate's reply is `/approve`. |
+| `MAINTAINERS`, `MAINTAINERS.md` (root, `.github/`, `docs/`) | `@handle` and `github.com/handle` spellings (plain list, markdown table or prose; names and e-mail addresses are not identities) | The hive-default permission-based `maintainer` role becomes kind `members` listing those handles (`source: path:line`); when no `approver` role exists, the review gate is released by `maintainer`. |
+| `kind/*`, `priority/*`, `type/*`, `area/*`, `severity/*`, `size/*` labels declared in `labels.yml`, `settings.yml`, `labeler.yml`, templates or backticked in `CONTRIBUTING.md` | Classification vocabulary triage applies | Plain `labels.entries` rows (no state), sorted by name, so the triage gate's vocabulary is visible in the editor. |
+| Any `approver` role | | Gate `review` released by `approver` (condition `pr.review is approved`) attached to the hive-default edges from CI / Review into the admitted Merge-queue states (`awaiting-merge-admission`, `ready-for-human-merge`, `pre-merge-soak`, `merge-queue`), so the compiler's "gate with no owner role" refusal is satisfied by a real role. |
+| Branch rules: `GET /repos/{o}/{r}/branches/{b}/protection`, falling back to `GET /repos/{o}/{r}/rules/branches/{b}` (repository and organisation rulesets); or probot `.github/settings.yml` `branches[].protection` when the API gave nothing | required status checks, required approving reviews, code-owner review, dismiss stale reviews, enforce admins, linear history, signed commits | `checks` entries `check-<slug>` of kind `ci`, `required: true`; guard `required-checks` (params `contexts`) on the same merge edges; `review.minApprovals: N`; gate `codeowner-review`; bookkeeping effects on `merged` (`enforce-admins`, `require-linear-history`, `require-signed-commits`, `dismiss-stale-reviews`) with the source in their params. |
+
+The forge half arrives as a synthetic file at `ingest.BranchRulesPath` (`.hive/branch-rules.json`,
+`ingest.BranchRules`): `cmd/hive`'s `workflowRepoFiles` fills it through `Client.FetchBranchRules`
+when ingesting by `repo`, so pasted `files` and the survey corpus can carry the same reading.
+`ingest.LifecycleOptions` accepts the rules explicitly, overriding the file.
+
+### Citations
+
+Every element cites `path:line` for file-derived facts (`Citation.Path`/`Line`; `checks[].source`,
+`guards[].source`, `roles[].params.source` and `bookkeeping[].params.source` carry the same
+`path:line` string) and `api:<endpoint>` — for example `api:repos/cncf/techdocs/rules/branches/main`
+— for facts read from the forge. `meta.source` becomes `issue-templates` (or
+`github-actions+issue-templates`) once a template was read.
+
+### What `cncf/techdocs` proposes
+
+With its current `main` (no issue templates yet; `.github/settings.yml` declares `needs-triage`,
+`triage/accepted`, `triage/declined`, `good first issue`, `help wanted`; the org ruleset requires no
+checks or approvals): columns `open · triage · actionable · claimed · pr-in-progress · ci-review ·
+merge-queue · merged`, the `triage` state citing `.github/settings.yml:129/132/135`, the three
+triage events, the `triage-decision` gate released by `maintainer`, and the five label rows. The
+intake form on the pending `build/03-intake` branch (`assessment-request.yml`, labels `Docs analysis`
++ `needs-triage`) adds `intake[requested-documentation-assessment-request]` before Triage — the
+fixture under `pkg/workflowmodel/ingest/testdata/techdocs` carries both.
+
+### Limitations
+
+- Heuristics prefer precision: a template without labels, or whose labels are all triage labels or
+  already bound to another intake state, yields a `issue-template-no-labels` / `issue-template-merged`
+  finding rather than a state. Label names are matched from a fixed table; a repository's own
+  triage word (`status: new`) is not recognised.
+- Only the root `OWNERS` is read (nested owners files scope by path, which the model does not
+  express); CODEOWNERS owners are all `approver`, since the file has no reviewer notion.
+- `codeowner-review` is informational: the engine does not observe which files a PR touches.
+- The engine still derives through the built-in stage table; nothing lands in Intake or Triage
+  until the label-driven engine half of phase 4 writes model labels onto items.
+- Branch protection needs `administration:read`; without it the rulesets endpoint is used, which
+  omits legacy protection settings such as `enforce_admins`.