diff --git a/README.md b/README.md index 7802bca0..9f8b1a8c 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Checks are organized in `check sets`. Each `check set` defines a number of check The CLOMonitor's linter can also be run locally or from CI workflows. This can be done by using the [container image](https://gallery.ecr.aws/clomonitor/linter) provided or by building the CLI tool from the source. -CLOMonitor delegates some of the security checks to [OpenSSF Scorecard](https://github.com/ossf/scorecard). When building from the source, you'll need to [install it](https://github.com/ossf/scorecard#installation) before running `clomonitor-linter` locally. The container image already includes the `scorecard` binary, so if you opt for using it you are ready to go. +CLOMonitor delegates some of the security checks to [OpenSSF Scorecard](https://github.com/ossf/scorecard) and the agent readiness checks to [AFDocs](https://afdocs.dev). When building from the source, you'll need to install them ([scorecard](https://github.com/ossf/scorecard#installation), [afdocs](https://www.npmjs.com/package/afdocs)) before running `clomonitor-linter` locally. The container image already includes both binaries, so if you opt for using it you are ready to go. Both CLOMonitor and Scorecard use the GitHub GraphQL API for some checks, which requires authentication. A GitHub token (with `public_repo` scope) **must** be provided via the `GITHUB_TOKEN` environment variable to authenticate those requests. diff --git a/clomonitor-apiserver/src/router.rs b/clomonitor-apiserver/src/router.rs index 24d324b3..3d1a8e07 100644 --- a/clomonitor-apiserver/src/router.rs +++ b/clomonitor-apiserver/src/router.rs @@ -723,9 +723,11 @@ mod tests { check_sets: vec![CheckSet::Code], score: Some(Score { global: 99.999_999_999_999_99, - global_weight: 5, + global_weight: 6, documentation: Some(100.0), documentation_weight: Some(1), + agent_readiness: Some(100.0), + agent_readiness_weight: Some(1), license: Some(100.0), license_weight: Some(1), best_practices: Some(100.0), @@ -748,6 +750,15 @@ mod tests { summary_table: Some(CheckOutput::passed()), website: Some(CheckOutput::passed()), }, + agent_readiness: AgentReadiness { + authentication: Some(CheckOutput::passed()), + content_discoverability: Some(CheckOutput::passed()), + content_structure: Some(CheckOutput::passed()), + markdown_availability: Some(CheckOutput::passed()), + observability: Some(CheckOutput::passed()), + page_size: Some(CheckOutput::passed()), + url_stability: Some(CheckOutput::passed()), + }, license: License { license_approved: Some(CheckOutput::passed()), license_scanning: Some( diff --git a/clomonitor-apiserver/src/testdata/repository-report.golden.md b/clomonitor-apiserver/src/testdata/repository-report.golden.md index 36c02585..4bb4b09e 100644 --- a/clomonitor-apiserver/src/testdata/repository-report.golden.md +++ b/clomonitor-apiserver/src/testdata/repository-report.golden.md @@ -9,13 +9,14 @@ ### Checks passed per category -| Category | Score | -| :------------- | ----------------------------------------------: | -| Documentation | 100% | -| License | 100% | -| Best Practices | 100% | -| Security | 100% | -| Legal | 100% | +| Category | Score | +| :-------------- | -----------------------------------------------: | +| Documentation | 100% | +| Agent Readiness | 100% | +| License | 100% | +| Best Practices | 100% | +| Security | 100% | +| Legal | 100% | ## Checks @@ -32,6 +33,16 @@ - [x] Summary Table ([_docs_](https://clomonitor.io/docs/topics/checks/#summary-table)) - [x] Website ([_docs_](https://clomonitor.io/docs/topics/checks/#website)) +### Agent Readiness [100%] + + - [x] Authentication ([_docs_](https://clomonitor.io/docs/topics/checks/#authentication)) + - [x] Content discoverability ([_docs_](https://clomonitor.io/docs/topics/checks/#content-discoverability)) + - [x] Content structure ([_docs_](https://clomonitor.io/docs/topics/checks/#content-structure)) + - [x] Markdown availability ([_docs_](https://clomonitor.io/docs/topics/checks/#markdown-availability)) + - [x] Observability ([_docs_](https://clomonitor.io/docs/topics/checks/#observability)) + - [x] Page size ([_docs_](https://clomonitor.io/docs/topics/checks/#page-size)) + - [x] URL stability ([_docs_](https://clomonitor.io/docs/topics/checks/#url-stability)) + ### License [100%] - [x] Apache-2.0 ([_docs_](https://clomonitor.io/docs/topics/checks/#spdx-id)) diff --git a/clomonitor-apiserver/templates/repository-report.md b/clomonitor-apiserver/templates/repository-report.md index 1d4ab912..1b86f32f 100644 --- a/clomonitor-apiserver/templates/repository-report.md +++ b/clomonitor-apiserver/templates/repository-report.md @@ -11,13 +11,14 @@ ### Checks passed per category -| Category | Score | -| :------------- | ----------------------------------------------: | -| Documentation | {{ category_score(score.documentation) }} | -| License | {{ category_score(score.license) }} | -| Best Practices | {{ category_score(score.best_practices) }} | -| Security | {{ category_score(score.security) }} | -| Legal | {{ category_score(score.legal) }} | +| Category | Score | +| :-------------- | -----------------------------------------------: | +| Documentation | {{ category_score(score.documentation) }} | +| Agent Readiness | {{ category_score(score.agent_readiness) }} | +| License | {{ category_score(score.license) }} | +| Best Practices | {{ category_score(score.best_practices) }} | +| Security | {{ category_score(score.security) }} | +| Legal | {{ category_score(score.legal) }} | ## Checks @@ -35,6 +36,18 @@ {{ check("summary-table", "Summary Table", report.documentation.summary_table) -}} {{ check("website", "Website", report.documentation.website) -}} +{%- endif %} +{%- if let Some(value) = score.agent_readiness %} +### Agent Readiness [{{ value.round() }}%] + + {{ check("authentication", "Authentication", report.agent_readiness.authentication) -}} + {{ check("content-discoverability", "Content discoverability", report.agent_readiness.content_discoverability) -}} + {{ check("content-structure", "Content structure", report.agent_readiness.content_structure) -}} + {{ check("markdown-availability", "Markdown availability", report.agent_readiness.markdown_availability) -}} + {{ check("observability", "Observability", report.agent_readiness.observability) -}} + {{ check("page-size", "Page size", report.agent_readiness.page_size) -}} + {{ check("url-stability", "URL stability", report.agent_readiness.url_stability) -}} + {%- endif %} {%- if let Some(value) = score.license %} ### License [{{ value.round() }}%] diff --git a/clomonitor-core/src/linter/check.rs b/clomonitor-core/src/linter/check.rs index 84e6c4b2..c581dbc8 100644 --- a/clomonitor-core/src/linter/check.rs +++ b/clomonitor-core/src/linter/check.rs @@ -4,8 +4,12 @@ use which::which; use super::{ CheckSet, LinterInput, - checks::{CHECKS, signed_releases}, + checks::{ + CHECKS, authentication, content_discoverability, content_structure, markdown_availability, + observability, page_size, signed_releases, url_stability, util::helpers::should_skip_check, + }, datasource::{ + afdocs::{self, AfdocsCheckResult, AfdocsCheckStatus, AfdocsReport}, github, scorecard::{Scorecard, ScorecardCheck, scorecard}, security_insights::SecurityInsights, @@ -31,6 +35,7 @@ pub(crate) struct CheckInput<'a> { pub gh_md: github::md::MdRepository, pub scorecard: Result, pub security_insights: Result>, + pub afdocs: Result>, } impl CheckInput<'_> { @@ -41,6 +46,11 @@ impl CheckInput<'_> { "scorecard not found in PATH (https://github.com/ossf/scorecard#installation)" )); } + if afdocs_required(&li.check_sets) && which("afdocs").is_err() { + return Err(format_err!( + "afdocs not found in PATH (https://www.npmjs.com/package/afdocs)" + )); + } // Get CLOMonitor metadata let cm_md = Metadata::from(li.root.join(METADATA_FILE))?; @@ -52,10 +62,28 @@ impl CheckInput<'_> { // Get GitHub metadata let gh_md = github::metadata(&li.url, &li.github_token).await?; - // Get OpenSSF scorecard - let scorecard = scorecard(&li.url, &li.github_token) - .await - .context("error running scorecard command"); + // Get OpenSSF scorecard and afdocs report. The afdocs command does not + // use the GitHub token, so running both concurrently cannot trigger the + // GitHub secondary rate limits mentioned above. The afdocs report is + // only fetched when any of the checks that rely on it will be run. + let (scorecard, afdocs) = tokio::join!( + async { + scorecard(&li.url, &li.github_token) + .await + .context("error running scorecard command") + }, + async { + match &gh_md.homepage_url { + Some(url) if !url.is_empty() && afdocs_required(&li.check_sets) => { + afdocs::afdocs(url) + .await + .map(Some) + .context("error running afdocs command") + } + _ => Ok(None), + } + } + ); // Get OpenSSF security insights. let security_insights = SecurityInsights::new(&li.root); @@ -67,11 +95,31 @@ impl CheckInput<'_> { gh_md, scorecard, security_insights, + afdocs, }; Ok(ci) } } +/// Check ids of the checks that rely on the afdocs report. +const AFDOCS_CHECKS: [CheckId; 7] = [ + authentication::ID, + content_discoverability::ID, + content_structure::ID, + markdown_availability::ID, + observability::ID, + page_size::ID, + url_stability::ID, +]; + +/// Check if any of the checks that rely on the afdocs report will be run for +/// the check sets provided. +fn afdocs_required(check_sets: &[CheckSet]) -> bool { + AFDOCS_CHECKS + .iter() + .any(|check_id| !should_skip_check(check_id, check_sets)) +} + /// Check output information. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CheckOutput { @@ -235,6 +283,77 @@ impl From, &Error>> for CheckOutput { } } +impl From>, &Error>> for CheckOutput { + fn from(results: Result>, &Error>) -> Self { + match results { + Ok(Some(results)) if !results.is_empty() => { + // Prepare category name from the category of the first result + let category_name = match results[0].category.as_str() { + "url-stability" => "URL stability".to_string(), + slug => { + let category = slug.replace('-', " "); + let mut chars = category.chars(); + match chars.next() { + Some(first) => { + first.to_uppercase().collect::() + chars.as_str() + } + None => String::new(), + } + } + }; + + // Prepare check output based on the results statuses + let all_skipped = results.iter().all(|r| r.status == AfdocsCheckStatus::Skip); + let some_failed = results.iter().any(|r| { + matches!(r.status, AfdocsCheckStatus::Fail | AfdocsCheckStatus::Error) + }); + let (mut output, result) = if all_skipped { + ( + CheckOutput::exempt().exemption_reason(Some( + "afdocs skipped all checks in this category for this website" + .to_string(), + )), + "skipped (afdocs skipped all checks in this category for this website)", + ) + } else if some_failed { + ( + CheckOutput::not_passed(), + "not passed (a category passes when none of its checks fail)", + ) + } else { + ( + CheckOutput::passed(), + "passed (a category passes when none of its checks fail)", + ) + }; + output.details = Some(format!( + r"# {} AFDocs checks + +**Result**: {} + +{} + +**Please see the [Agent-Friendly Documentation Spec]({}) for more details**", + category_name, + result, + results + .iter() + .map(|r| format!("- **{}** `{}`: {}", r.status, r.id, r.message)) + .collect::>() + .join("\n"), + afdocs::SPEC_URL, + )); + output + } + Ok(_) => CheckOutput::failed().fail_reason(Some( + "category not found in the afdocs report (possible afdocs version mismatch)" + .to_string(), + )), + Err(err) => CheckOutput::failed().fail_reason(Some(format!("{err:#}"))), + } + } +} + /// Wrapper macro that takes care of running some common pre-check operations /// and the synchronous check function. macro_rules! run { @@ -356,8 +475,10 @@ mod tests { #[test] fn check_output_from_scorecard_check_not_available() { + let sc_check: Result, &Error> = Ok(None); + assert_eq!( - CheckOutput::<()>::from(Ok(None)), + CheckOutput::<()>::from(sc_check), CheckOutput { passed: false, ..Default::default() @@ -379,4 +500,172 @@ mod tests { } ); } + + #[test] + fn check_output_from_afdocs_category_passed() { + let results = [ + AfdocsCheckResult { + id: "markdown-url-support".to_string(), + category: "markdown-availability".to_string(), + status: AfdocsCheckStatus::Pass, + message: "Markdown URLs supported".to_string(), + }, + AfdocsCheckResult { + id: "content-negotiation".to_string(), + category: "markdown-availability".to_string(), + status: AfdocsCheckStatus::Warn, + message: "partial support".to_string(), + }, + ]; + let results: Result>, &Error> = + Ok(Some(results.iter().collect())); + + assert_eq!( + CheckOutput::<()>::from(results), + CheckOutput { + passed: true, + details: Some("# Markdown availability AFDocs checks\n\n**Result**: passed (a category passes when none of its checks fail)\n\n- **PASS** `markdown-url-support`: Markdown URLs supported\n- **WARN** `content-negotiation`: partial support\n\n**Please see the [Agent-Friendly Documentation Spec](https://agentdocsspec.com/spec/) for more details**".to_string()), + ..Default::default() + } + ); + } + + #[test] + fn check_output_from_afdocs_category_not_passed() { + let results = [ + AfdocsCheckResult { + id: "llms-txt-exists".to_string(), + category: "content-discoverability".to_string(), + status: AfdocsCheckStatus::Warn, + message: "llms.txt found but only reachable via cross-host redirect".to_string(), + }, + AfdocsCheckResult { + id: "llms-txt-directive-html".to_string(), + category: "content-discoverability".to_string(), + status: AfdocsCheckStatus::Fail, + message: "No llms.txt directive found in HTML of any of 1 pages".to_string(), + }, + ]; + let results: Result>, &Error> = + Ok(Some(results.iter().collect())); + + assert_eq!( + CheckOutput::<()>::from(results), + CheckOutput { + passed: false, + details: Some("# Content discoverability AFDocs checks\n\n**Result**: not passed (a category passes when none of its checks fail)\n\n- **WARN** `llms-txt-exists`: llms.txt found but only reachable via cross-host redirect\n- **FAIL** `llms-txt-directive-html`: No llms.txt directive found in HTML of any of 1 pages\n\n**Please see the [Agent-Friendly Documentation Spec](https://agentdocsspec.com/spec/) for more details**".to_string()), + ..Default::default() + } + ); + } + + #[test] + fn check_output_from_afdocs_category_error_not_passed() { + let results = [AfdocsCheckResult { + id: "http-status-codes".to_string(), + category: "url-stability".to_string(), + status: AfdocsCheckStatus::Error, + message: "check errored".to_string(), + }]; + let results: Result>, &Error> = + Ok(Some(results.iter().collect())); + + let output = CheckOutput::<()>::from(results); + assert!(!output.passed); + assert!(!output.exempt); + assert!( + output + .details + .unwrap() + .starts_with("# URL stability AFDocs checks") + ); + } + + #[test] + fn check_output_from_afdocs_category_all_skipped() { + let results = [AfdocsCheckResult { + id: "llms-txt-coverage".to_string(), + category: "observability".to_string(), + status: AfdocsCheckStatus::Skip, + message: "llms.txt not found".to_string(), + }]; + let results: Result>, &Error> = + Ok(Some(results.iter().collect())); + + assert_eq!( + CheckOutput::<()>::from(results), + CheckOutput { + exempt: true, + exemption_reason: Some( + "afdocs skipped all checks in this category for this website".to_string() + ), + details: Some("# Observability AFDocs checks\n\n**Result**: skipped (afdocs skipped all checks in this category for this website)\n\n- **SKIP** `llms-txt-coverage`: llms.txt not found\n\n**Please see the [Agent-Friendly Documentation Spec](https://agentdocsspec.com/spec/) for more details**".to_string()), + ..Default::default() + } + ); + } + + #[test] + fn check_output_from_afdocs_category_not_available() { + let results: Result>, &Error> = Ok(None); + + assert_eq!( + CheckOutput::<()>::from(results), + CheckOutput { + failed: true, + fail_reason: Some( + "category not found in the afdocs report (possible afdocs version mismatch)" + .to_string() + ), + ..Default::default() + } + ); + } + + #[test] + fn check_output_from_afdocs_category_empty_results() { + let results: Result>, &Error> = Ok(Some(vec![])); + + assert_eq!( + CheckOutput::<()>::from(results), + CheckOutput { + failed: true, + fail_reason: Some( + "category not found in the afdocs report (possible afdocs version mismatch)" + .to_string() + ), + ..Default::default() + } + ); + } + + #[test] + fn check_output_from_afdocs_category_failed() { + let err = format_err!("fake error"); + let results: Result>, &Error> = Err(&err); + + assert_eq!( + CheckOutput::<()>::from(results), + CheckOutput { + failed: true, + fail_reason: Some("fake error".to_string()), + ..Default::default() + } + ); + } + + #[test] + fn afdocs_required_matching_check_sets() { + assert!(afdocs_required(&[CheckSet::Community])); + assert!(afdocs_required(&[CheckSet::Docs])); + assert!(afdocs_required(&[CheckSet::Code, CheckSet::Community])); + } + + #[test] + fn afdocs_required_no_matching_check_sets() { + assert!(!afdocs_required(&[CheckSet::Code])); + assert!(!afdocs_required(&[CheckSet::CodeLite])); + assert!(!afdocs_required(&[CheckSet::Code, CheckSet::CodeLite])); + assert!(!afdocs_required(&[])); + } } diff --git a/clomonitor-core/src/linter/checks/authentication.rs b/clomonitor-core/src/linter/checks/authentication.rs new file mode 100644 index 00000000..bb62f487 --- /dev/null +++ b/clomonitor-core/src/linter/checks/authentication.rs @@ -0,0 +1,23 @@ +use anyhow::Result; + +use crate::linter::{ + CheckSet, + check::{CheckId, CheckInput, CheckOutput}, +}; + +use super::datasource::afdocs; + +/// Check identifier. +pub(crate) const ID: CheckId = "authentication"; + +/// Check score weight. +pub(crate) const WEIGHT: usize = 1; + +/// Check sets this check belongs to. +pub(crate) const CHECK_SETS: [CheckSet; 2] = [CheckSet::Community, CheckSet::Docs]; + +/// Check main function. +#[allow(clippy::unnecessary_wraps)] +pub(crate) fn check(input: &CheckInput) -> Result { + Ok(afdocs::get_category(&input.afdocs, ID).into()) +} diff --git a/clomonitor-core/src/linter/checks/content_discoverability.rs b/clomonitor-core/src/linter/checks/content_discoverability.rs new file mode 100644 index 00000000..0dc1b276 --- /dev/null +++ b/clomonitor-core/src/linter/checks/content_discoverability.rs @@ -0,0 +1,23 @@ +use anyhow::Result; + +use crate::linter::{ + CheckSet, + check::{CheckId, CheckInput, CheckOutput}, +}; + +use super::datasource::afdocs; + +/// Check identifier. +pub(crate) const ID: CheckId = "content_discoverability"; + +/// Check score weight. +pub(crate) const WEIGHT: usize = 1; + +/// Check sets this check belongs to. +pub(crate) const CHECK_SETS: [CheckSet; 2] = [CheckSet::Community, CheckSet::Docs]; + +/// Check main function. +#[allow(clippy::unnecessary_wraps)] +pub(crate) fn check(input: &CheckInput) -> Result { + Ok(afdocs::get_category(&input.afdocs, ID).into()) +} diff --git a/clomonitor-core/src/linter/checks/content_structure.rs b/clomonitor-core/src/linter/checks/content_structure.rs new file mode 100644 index 00000000..a041fec5 --- /dev/null +++ b/clomonitor-core/src/linter/checks/content_structure.rs @@ -0,0 +1,23 @@ +use anyhow::Result; + +use crate::linter::{ + CheckSet, + check::{CheckId, CheckInput, CheckOutput}, +}; + +use super::datasource::afdocs; + +/// Check identifier. +pub(crate) const ID: CheckId = "content_structure"; + +/// Check score weight. +pub(crate) const WEIGHT: usize = 1; + +/// Check sets this check belongs to. +pub(crate) const CHECK_SETS: [CheckSet; 2] = [CheckSet::Community, CheckSet::Docs]; + +/// Check main function. +#[allow(clippy::unnecessary_wraps)] +pub(crate) fn check(input: &CheckInput) -> Result { + Ok(afdocs::get_category(&input.afdocs, ID).into()) +} diff --git a/clomonitor-core/src/linter/checks/datasource/afdocs.rs b/clomonitor-core/src/linter/checks/datasource/afdocs.rs new file mode 100644 index 00000000..49d62346 --- /dev/null +++ b/clomonitor-core/src/linter/checks/datasource/afdocs.rs @@ -0,0 +1,181 @@ +use std::fmt; + +use anyhow::{Error, Result, format_err}; +use serde::Deserialize; +use tokio::process::Command; + +/// Agent-Friendly Documentation Spec url. +pub(crate) const SPEC_URL: &str = "https://agentdocsspec.com/spec/"; + +/// AFDocs report (list of check results). +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct AfdocsReport { + results: Vec, +} + +/// AFDocs check result details. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +pub(crate) struct AfdocsCheckResult { + pub id: String, + pub category: String, + pub status: AfdocsCheckStatus, + pub message: String, +} + +/// AFDocs check result status. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub(crate) enum AfdocsCheckStatus { + Pass, + Warn, + Fail, + Skip, + Error, + #[serde(other)] + Other, +} + +impl fmt::Display for AfdocsCheckStatus { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let status = match self { + Self::Pass => "PASS", + Self::Warn => "WARN", + Self::Fail => "FAIL", + Self::Skip => "SKIP", + Self::Error => "ERROR", + Self::Other => "OTHER", + }; + write!(f, "{status}") + } +} + +/// Run afdocs on the website provided and return its report. The afdocs +/// command exits with a non-zero code when any of its checks fail, so the +/// exit code is ignored as long as the output contains a valid report. +pub(crate) async fn afdocs(website_url: &str) -> Result { + let output = Command::new("afdocs") + .arg("check") + .arg(website_url) + .arg("--format") + .arg("json") + .output() + .await?; + let stdout = String::from_utf8_lossy(&output.stdout); + let report: AfdocsReport = serde_json::from_str(stdout.as_ref()).map_err(|_| { + format_err!( + "unexpected afdocs output: {}", + String::from_utf8_lossy(&output.stderr) + ) + })?; + Ok(report) +} + +/// Get all check results in the afdocs report that belong to the category +/// the check provided represents. +pub(crate) fn get_category<'a>( + afdocs: &'a Result>, + check_id: &str, +) -> Result>, &'a Error> { + let category = check_id.replace('_', "-"); + match afdocs { + Ok(Some(report)) => { + let results: Vec<&AfdocsCheckResult> = report + .results + .iter() + .filter(|r| r.category == category) + .collect(); + if results.is_empty() { + return Ok(None); + } + Ok(Some(results)) + } + Ok(None) => Ok(None), + Err(err) => Err(err), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + pub(crate) fn sample_results() -> Vec { + vec![ + AfdocsCheckResult { + id: "llms-txt-exists".to_string(), + category: "content-discoverability".to_string(), + status: AfdocsCheckStatus::Warn, + message: "llms.txt found but only reachable via cross-host redirect".to_string(), + }, + AfdocsCheckResult { + id: "llms-txt-directive-html".to_string(), + category: "content-discoverability".to_string(), + status: AfdocsCheckStatus::Fail, + message: "No llms.txt directive found in HTML of any of 1 pages".to_string(), + }, + AfdocsCheckResult { + id: "markdown-url-support".to_string(), + category: "markdown-availability".to_string(), + status: AfdocsCheckStatus::Pass, + message: "Markdown URLs supported".to_string(), + }, + ] + } + + #[test] + fn report_parsed_from_real_json_output() { + let json = std::fs::read_to_string("src/testdata/afdocs/report.json").unwrap(); + let report: AfdocsReport = serde_json::from_str(&json).unwrap(); + + assert_eq!(report.results.len(), 23); + assert!( + report + .results + .iter() + .any(|r| r.category == "content-discoverability") + ); + } + + #[test] + fn get_category_found() { + let afdocs = Ok(Some(AfdocsReport { + results: sample_results(), + })); + + let results = get_category(&afdocs, "content_discoverability") + .unwrap() + .unwrap(); + assert_eq!(results.len(), 2); + assert!( + results + .iter() + .all(|r| r.category == "content-discoverability") + ); + } + + #[test] + fn get_category_not_found() { + let afdocs = Ok(Some(AfdocsReport { + results: sample_results(), + })); + + assert!(get_category(&afdocs, "page_size").unwrap().is_none()); + } + + #[test] + fn get_category_no_website() { + let afdocs = Ok(None); + + assert!( + get_category(&afdocs, "content_discoverability") + .unwrap() + .is_none() + ); + } + + #[test] + fn get_category_datasource_error() { + let afdocs = Err(format_err!("something failed")); + + assert!(get_category(&afdocs, "content_discoverability").is_err()); + } +} diff --git a/clomonitor-core/src/linter/checks/datasource/mod.rs b/clomonitor-core/src/linter/checks/datasource/mod.rs index ef193c77..c4c4e86b 100644 --- a/clomonitor-core/src/linter/checks/datasource/mod.rs +++ b/clomonitor-core/src/linter/checks/datasource/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod afdocs; pub(crate) mod github; pub(crate) mod landscape; pub(crate) mod scorecard; diff --git a/clomonitor-core/src/linter/checks/dependencies_policy.rs b/clomonitor-core/src/linter/checks/dependencies_policy.rs index 399536e1..987a883b 100644 --- a/clomonitor-core/src/linter/checks/dependencies_policy.rs +++ b/clomonitor-core/src/linter/checks/dependencies_policy.rs @@ -50,6 +50,7 @@ mod tests { .canonicalize() .unwrap(), ), + afdocs: Ok(None), }) .unwrap(); @@ -68,6 +69,7 @@ mod tests { .canonicalize() .unwrap(), ), + afdocs: Ok(None), }) .unwrap(); @@ -91,6 +93,7 @@ mod tests { .canonicalize() .unwrap(), ), + afdocs: Ok(None), }) .unwrap(); diff --git a/clomonitor-core/src/linter/checks/github_discussions.rs b/clomonitor-core/src/linter/checks/github_discussions.rs index e2d9ab65..c904a78c 100644 --- a/clomonitor-core/src/linter/checks/github_discussions.rs +++ b/clomonitor-core/src/linter/checks/github_discussions.rs @@ -58,6 +58,7 @@ mod tests { }, scorecard: Err(format_err!("no scorecard available")), security_insights: Ok(None), + afdocs: Ok(None), }) .unwrap(), CheckOutput::not_passed(), @@ -85,6 +86,7 @@ mod tests { }, scorecard: Err(format_err!("no scorecard available")), security_insights: Ok(None), + afdocs: Ok(None), }) .unwrap(), CheckOutput::not_passed(), @@ -112,6 +114,7 @@ mod tests { }, scorecard: Err(format_err!("no scorecard available")), security_insights: Ok(None), + afdocs: Ok(None), }) .unwrap(), CheckOutput::passed().url(Some("discussion_url".to_string())), diff --git a/clomonitor-core/src/linter/checks/license_scanning.rs b/clomonitor-core/src/linter/checks/license_scanning.rs index 5487e70f..ab519131 100644 --- a/clomonitor-core/src/linter/checks/license_scanning.rs +++ b/clomonitor-core/src/linter/checks/license_scanning.rs @@ -70,6 +70,7 @@ mod tests { gh_md: MdRepository::default(), scorecard: Err(format_err!("no scorecard available")), security_insights: Ok(None), + afdocs: Ok(None), }) .unwrap(), CheckOutput::not_passed(), @@ -88,6 +89,7 @@ mod tests { gh_md: MdRepository::default(), scorecard: Err(format_err!("no scorecard available")), security_insights: Ok(None), + afdocs: Ok(None), }) .unwrap(), CheckOutput::not_passed(), @@ -108,6 +110,7 @@ mod tests { gh_md: MdRepository::default(), scorecard: Err(format_err!("no scorecard available")), security_insights: Ok(None), + afdocs: Ok(None), }) .unwrap(), CheckOutput::passed().url(Some("license_scanning_url".to_string())), diff --git a/clomonitor-core/src/linter/checks/markdown_availability.rs b/clomonitor-core/src/linter/checks/markdown_availability.rs new file mode 100644 index 00000000..c62f6c59 --- /dev/null +++ b/clomonitor-core/src/linter/checks/markdown_availability.rs @@ -0,0 +1,23 @@ +use anyhow::Result; + +use crate::linter::{ + CheckSet, + check::{CheckId, CheckInput, CheckOutput}, +}; + +use super::datasource::afdocs; + +/// Check identifier. +pub(crate) const ID: CheckId = "markdown_availability"; + +/// Check score weight. +pub(crate) const WEIGHT: usize = 1; + +/// Check sets this check belongs to. +pub(crate) const CHECK_SETS: [CheckSet; 2] = [CheckSet::Community, CheckSet::Docs]; + +/// Check main function. +#[allow(clippy::unnecessary_wraps)] +pub(crate) fn check(input: &CheckInput) -> Result { + Ok(afdocs::get_category(&input.afdocs, ID).into()) +} diff --git a/clomonitor-core/src/linter/checks/mod.rs b/clomonitor-core/src/linter/checks/mod.rs index 5ca563d6..82ac4d7d 100644 --- a/clomonitor-core/src/linter/checks/mod.rs +++ b/clomonitor-core/src/linter/checks/mod.rs @@ -5,12 +5,15 @@ use crate::linter::check::{CheckConfig, CheckId}; pub(crate) mod adopters; pub(crate) mod analytics; pub(crate) mod artifacthub_badge; +pub(crate) mod authentication; pub(crate) mod binary_artifacts; pub(crate) mod changelog; pub(crate) mod cla; pub(crate) mod code_of_conduct; pub(crate) mod code_review; pub(crate) mod community_meeting; +pub(crate) mod content_discoverability; +pub(crate) mod content_structure; pub(crate) mod contributing; pub(crate) mod dangerous_workflow; pub(crate) mod datasource; @@ -24,8 +27,11 @@ pub(crate) mod license_scanning; pub(crate) mod license_spdx_id; pub(crate) mod maintained; pub(crate) mod maintainers; +pub(crate) mod markdown_availability; +pub(crate) mod observability; pub(crate) mod openssf_badge; pub(crate) mod openssf_scorecard_badge; +pub(crate) mod page_size; pub(crate) mod readme; pub(crate) mod recent_release; pub(crate) mod roadmap; @@ -37,6 +43,7 @@ pub(crate) mod slack_presence; pub(crate) mod summary_table; pub(crate) mod token_permissions; pub(crate) mod trademark_disclaimer; +pub(crate) mod url_stability; pub(crate) mod util; pub(crate) mod website; @@ -69,12 +76,15 @@ pub(crate) static CHECKS: LazyLock> = LazyLock::ne register_check!(adopters); register_check!(analytics); register_check!(artifacthub_badge); + register_check!(authentication); register_check!(binary_artifacts, "Binary-Artifacts"); register_check!(changelog); register_check!(cla); register_check!(code_of_conduct); register_check!(code_review, "Code-Review"); register_check!(community_meeting); + register_check!(content_discoverability); + register_check!(content_structure); register_check!(contributing); register_check!(dangerous_workflow, "Dangerous-Workflow"); register_check!(dco); @@ -87,8 +97,11 @@ pub(crate) static CHECKS: LazyLock> = LazyLock::ne register_check!(license_spdx_id); register_check!(maintained, "Maintained"); register_check!(maintainers); + register_check!(markdown_availability); + register_check!(observability); register_check!(openssf_badge); register_check!(openssf_scorecard_badge); + register_check!(page_size); register_check!(readme); register_check!(recent_release); register_check!(roadmap); @@ -100,6 +113,7 @@ pub(crate) static CHECKS: LazyLock> = LazyLock::ne register_check!(summary_table); register_check!(token_permissions, "Token-Permissions"); register_check!(trademark_disclaimer); + register_check!(url_stability); register_check!(website); checks diff --git a/clomonitor-core/src/linter/checks/observability.rs b/clomonitor-core/src/linter/checks/observability.rs new file mode 100644 index 00000000..80b6abb2 --- /dev/null +++ b/clomonitor-core/src/linter/checks/observability.rs @@ -0,0 +1,23 @@ +use anyhow::Result; + +use crate::linter::{ + CheckSet, + check::{CheckId, CheckInput, CheckOutput}, +}; + +use super::datasource::afdocs; + +/// Check identifier. +pub(crate) const ID: CheckId = "observability"; + +/// Check score weight. +pub(crate) const WEIGHT: usize = 1; + +/// Check sets this check belongs to. +pub(crate) const CHECK_SETS: [CheckSet; 2] = [CheckSet::Community, CheckSet::Docs]; + +/// Check main function. +#[allow(clippy::unnecessary_wraps)] +pub(crate) fn check(input: &CheckInput) -> Result { + Ok(afdocs::get_category(&input.afdocs, ID).into()) +} diff --git a/clomonitor-core/src/linter/checks/page_size.rs b/clomonitor-core/src/linter/checks/page_size.rs new file mode 100644 index 00000000..8ccb5ac9 --- /dev/null +++ b/clomonitor-core/src/linter/checks/page_size.rs @@ -0,0 +1,23 @@ +use anyhow::Result; + +use crate::linter::{ + CheckSet, + check::{CheckId, CheckInput, CheckOutput}, +}; + +use super::datasource::afdocs; + +/// Check identifier. +pub(crate) const ID: CheckId = "page_size"; + +/// Check score weight. +pub(crate) const WEIGHT: usize = 1; + +/// Check sets this check belongs to. +pub(crate) const CHECK_SETS: [CheckSet; 2] = [CheckSet::Community, CheckSet::Docs]; + +/// Check main function. +#[allow(clippy::unnecessary_wraps)] +pub(crate) fn check(input: &CheckInput) -> Result { + Ok(afdocs::get_category(&input.afdocs, ID).into()) +} diff --git a/clomonitor-core/src/linter/checks/recent_release.rs b/clomonitor-core/src/linter/checks/recent_release.rs index bb51d41b..112c3ed4 100644 --- a/clomonitor-core/src/linter/checks/recent_release.rs +++ b/clomonitor-core/src/linter/checks/recent_release.rs @@ -56,6 +56,7 @@ mod tests { }, scorecard: Err(format_err!("no scorecard available")), security_insights: Ok(None), + afdocs: Ok(None), }) .unwrap(), CheckOutput::not_passed(), @@ -87,6 +88,7 @@ mod tests { }, scorecard: Err(format_err!("no scorecard available")), security_insights: Ok(None), + afdocs: Ok(None), }) .unwrap(), CheckOutput::not_passed(), @@ -118,6 +120,7 @@ mod tests { }, scorecard: Err(format_err!("no scorecard available")), security_insights: Ok(None), + afdocs: Ok(None), }) .unwrap(), CheckOutput::passed().url(Some("release_url".to_string())), diff --git a/clomonitor-core/src/linter/checks/sbom.rs b/clomonitor-core/src/linter/checks/sbom.rs index 4c6fcbe9..b7cb4a3e 100644 --- a/clomonitor-core/src/linter/checks/sbom.rs +++ b/clomonitor-core/src/linter/checks/sbom.rs @@ -80,6 +80,7 @@ mod tests { }, scorecard: Err(format_err!("no scorecard available")), security_insights: Ok(None), + afdocs: Ok(None), }) .unwrap(), CheckOutput::not_passed(), @@ -113,6 +114,7 @@ mod tests { }, scorecard: Err(format_err!("no scorecard available")), security_insights: Ok(None), + afdocs: Ok(None), }) .unwrap(), CheckOutput::not_passed(), @@ -146,6 +148,7 @@ mod tests { }, scorecard: Err(format_err!("no scorecard available")), security_insights: Ok(None), + afdocs: Ok(None), }) .unwrap(), CheckOutput::passed(), diff --git a/clomonitor-core/src/linter/checks/security_insights.rs b/clomonitor-core/src/linter/checks/security_insights.rs index 5bf0c269..cbf9b05e 100644 --- a/clomonitor-core/src/linter/checks/security_insights.rs +++ b/clomonitor-core/src/linter/checks/security_insights.rs @@ -60,6 +60,7 @@ mod tests { .canonicalize() .unwrap(), ), + afdocs: Ok(None), }) .unwrap(); @@ -83,6 +84,7 @@ mod tests { .canonicalize() .unwrap(), ), + afdocs: Ok(None), }) .unwrap(); @@ -106,6 +108,7 @@ mod tests { .canonicalize() .unwrap(), ), + afdocs: Ok(None), }) .unwrap(); diff --git a/clomonitor-core/src/linter/checks/url_stability.rs b/clomonitor-core/src/linter/checks/url_stability.rs new file mode 100644 index 00000000..e3f65830 --- /dev/null +++ b/clomonitor-core/src/linter/checks/url_stability.rs @@ -0,0 +1,23 @@ +use anyhow::Result; + +use crate::linter::{ + CheckSet, + check::{CheckId, CheckInput, CheckOutput}, +}; + +use super::datasource::afdocs; + +/// Check identifier. +pub(crate) const ID: CheckId = "url_stability"; + +/// Check score weight. +pub(crate) const WEIGHT: usize = 1; + +/// Check sets this check belongs to. +pub(crate) const CHECK_SETS: [CheckSet; 2] = [CheckSet::Community, CheckSet::Docs]; + +/// Check main function. +#[allow(clippy::unnecessary_wraps)] +pub(crate) fn check(input: &CheckInput) -> Result { + Ok(afdocs::get_category(&input.afdocs, ID).into()) +} diff --git a/clomonitor-core/src/linter/checks/util/helpers.rs b/clomonitor-core/src/linter/checks/util/helpers.rs index 91265c84..e2217783 100644 --- a/clomonitor-core/src/linter/checks/util/helpers.rs +++ b/clomonitor-core/src/linter/checks/util/helpers.rs @@ -135,6 +135,7 @@ mod tests { }, scorecard: Err(format_err!("no scorecard available")), security_insights: Ok(None), + afdocs: Ok(None), }, &["README*"], &RegexSet::new(["nothing"]).unwrap(), @@ -159,6 +160,7 @@ mod tests { gh_md: MdRepository::default(), scorecard: Err(format_err!("no scorecard available")), security_insights: Ok(None), + afdocs: Ok(None), }, &["ADOPTERS*"], &RegexSet::new([r"(?im)^#+.*adopters.*$"]).unwrap(), @@ -181,6 +183,7 @@ mod tests { gh_md: MdRepository::default(), scorecard: Err(format_err!("no scorecard available")), security_insights: Ok(None), + afdocs: Ok(None), }, &["inexistent_file*"], &RegexSet::new(["inexistent_ref"]).unwrap(), diff --git a/clomonitor-core/src/linter/mod.rs b/clomonitor-core/src/linter/mod.rs index 1d395dfb..7a457195 100644 --- a/clomonitor-core/src/linter/mod.rs +++ b/clomonitor-core/src/linter/mod.rs @@ -138,6 +138,21 @@ impl Linter for CoreLinter { summary_table, website: run!(website, &ci), }, + // When the repository has no website configured, the afdocs report + // is not available and this section is omitted from the results. + agent_readiness: if matches!(ci.afdocs, Ok(None)) { + AgentReadiness::default() + } else { + AgentReadiness { + authentication: run!(authentication, &ci), + content_discoverability: run!(content_discoverability, &ci), + content_structure: run!(content_structure, &ci), + markdown_availability: run!(markdown_availability, &ci), + observability: run!(observability, &ci), + page_size: run!(page_size, &ci), + url_stability: run!(url_stability, &ci), + } + }, license: License { license_approved: license_approved::check(&ci, spdx_id_value), license_scanning: run!(license_scanning, &ci), diff --git a/clomonitor-core/src/linter/report.rs b/clomonitor-core/src/linter/report.rs index 6c42bab7..37188cfa 100644 --- a/clomonitor-core/src/linter/report.rs +++ b/clomonitor-core/src/linter/report.rs @@ -6,6 +6,8 @@ use super::{CheckOutput, check::CheckId, checks::*}; #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct Report { pub documentation: Documentation, + #[serde(default)] + pub agent_readiness: AgentReadiness, pub license: License, pub best_practices: BestPractices, pub security: Security, @@ -82,6 +84,30 @@ section_impl!( website ); +/// AgentReadiness section of the report. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct AgentReadiness { + pub authentication: Option, + pub content_discoverability: Option, + pub content_structure: Option, + pub markdown_availability: Option, + pub observability: Option, + pub page_size: Option, + pub url_stability: Option, +} + +#[rustfmt::skip] +section_impl!( + AgentReadiness, + authentication, + content_discoverability, + content_structure, + markdown_availability, + observability, + page_size, + url_stability +); + /// License section of the report. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct License { diff --git a/clomonitor-core/src/score/mod.rs b/clomonitor-core/src/score/mod.rs index 2b60ccde..c2fb4b37 100644 --- a/clomonitor-core/src/score/mod.rs +++ b/clomonitor-core/src/score/mod.rs @@ -14,6 +14,12 @@ pub struct Score { #[serde(skip_serializing_if = "Option::is_none")] pub documentation_weight: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub agent_readiness: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub agent_readiness_weight: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub license: Option, @@ -63,6 +69,10 @@ pub fn calculate(report: &Report) -> Score { &report.documentation.available(), &report.documentation.passed_or_exempt(), ); + (score.agent_readiness, score.agent_readiness_weight) = calculate_section( + &report.agent_readiness.available(), + &report.agent_readiness.passed_or_exempt(), + ); (score.license, score.license_weight) = calculate_section( &report.license.available(), &report.license.passed_or_exempt(), @@ -81,6 +91,7 @@ pub fn calculate(report: &Report) -> Score { // Global let sections_scores = &[ score.documentation, + score.agent_readiness, score.license, score.best_practices, score.security, @@ -88,6 +99,7 @@ pub fn calculate(report: &Report) -> Score { ]; let sections_weights = &[ score.documentation_weight, + score.agent_readiness_weight, score.license_weight, score.best_practices_weight, score.security_weight, @@ -135,6 +147,7 @@ pub fn merge(scores: &[Score]) -> Score { // calculate the coefficient we'll apply to each of the scores. let mut global_weights_sum = 0; let mut documentation_weights_sum = 0; + let mut agent_readiness_weights_sum = 0; let mut license_weights_sum = 0; let mut best_practices_weights_sum = 0; let mut security_weights_sum = 0; @@ -142,6 +155,7 @@ pub fn merge(scores: &[Score]) -> Score { for score in scores { global_weights_sum += score.global_weight; documentation_weights_sum += score.documentation_weight.unwrap_or_default(); + agent_readiness_weights_sum += score.agent_readiness_weight.unwrap_or_default(); license_weights_sum += score.license_weight.unwrap_or_default(); best_practices_weights_sum += score.best_practices_weight.unwrap_or_default(); security_weights_sum += score.security_weight.unwrap_or_default(); @@ -169,6 +183,12 @@ pub fn merge(scores: &[Score]) -> Score { s.documentation, s.documentation_weight.unwrap_or_default() as f64 / documentation_weights_sum as f64, ); + m.agent_readiness = merge( + m.agent_readiness, + s.agent_readiness, + s.agent_readiness_weight.unwrap_or_default() as f64 + / agent_readiness_weights_sum as f64, + ); m.license = merge( m.license, s.license, @@ -263,6 +283,15 @@ mod tests { summary_table: Some(CheckOutput::passed()), website: Some(CheckOutput::passed()), }, + agent_readiness: AgentReadiness { + authentication: Some(CheckOutput::passed()), + content_discoverability: Some(CheckOutput::passed()), + content_structure: Some(CheckOutput::passed()), + markdown_availability: Some(CheckOutput::passed()), + observability: Some(CheckOutput::passed()), + page_size: Some(CheckOutput::passed()), + url_stability: Some(CheckOutput::passed()), + }, license: License { license_approved: Some(CheckOutput::passed()), license_scanning: Some( @@ -302,10 +331,12 @@ mod tests { }, }), Score { - global: 100.0, - global_weight: 96, + global: 99.999_999_999_999_99, + global_weight: 103, documentation: Some(99.999_999_999_999_99), documentation_weight: Some(30), + agent_readiness: Some(99.999_999_999_999_97), + agent_readiness_weight: Some(7), license: Some(100.0), license_weight: Some(20), best_practices: Some(99.999_999_999_999_99), @@ -334,6 +365,15 @@ mod tests { summary_table: Some(CheckOutput::not_passed()), website: Some(CheckOutput::not_passed()), }, + agent_readiness: AgentReadiness { + authentication: Some(CheckOutput::not_passed()), + content_discoverability: Some(CheckOutput::not_passed()), + content_structure: Some(CheckOutput::not_passed()), + markdown_availability: Some(CheckOutput::not_passed()), + observability: Some(CheckOutput::not_passed()), + page_size: Some(CheckOutput::not_passed()), + url_stability: Some(CheckOutput::not_passed()), + }, license: License { license_approved: Some(CheckOutput::not_passed()), license_scanning: Some(CheckOutput::not_passed()), @@ -370,9 +410,11 @@ mod tests { }), Score { global: 0.0, - global_weight: 96, + global_weight: 103, documentation: Some(0.0), documentation_weight: Some(30), + agent_readiness: Some(0.0), + agent_readiness_weight: Some(7), license: Some(0.0), license_weight: Some(20), best_practices: Some(0.0), @@ -401,6 +443,7 @@ mod tests { summary_table: None, website: None, }, + agent_readiness: AgentReadiness::default(), license: License { license_approved: Some(CheckOutput::passed()), license_scanning: Some( @@ -444,6 +487,8 @@ mod tests { global_weight: 76, documentation: Some(100.0), documentation_weight: Some(18), + agent_readiness: None, + agent_readiness_weight: None, license: Some(100.0), license_weight: Some(20), best_practices: Some(100.0), @@ -465,6 +510,8 @@ mod tests { global_weight: 90, documentation: Some(100.0), documentation_weight: Some(30), + agent_readiness: Some(100.0), + agent_readiness_weight: Some(7), license: Some(100.0), license_weight: Some(20), best_practices: Some(100.0), @@ -479,6 +526,8 @@ mod tests { global_weight: 45, documentation: Some(0.0), documentation_weight: Some(15), + agent_readiness: Some(0.0), + agent_readiness_weight: Some(7), license: Some(0.0), license_weight: Some(10), best_practices: Some(0.0), @@ -494,6 +543,8 @@ mod tests { global_weight: 0, documentation: Some(66.666_666_666_666_66), documentation_weight: None, + agent_readiness: Some(50.0), + agent_readiness_weight: None, license: Some(66.666_666_666_666_66), license_weight: None, best_practices: Some(66.666_666_666_666_66), diff --git a/clomonitor-core/src/testdata/afdocs/report.json b/clomonitor-core/src/testdata/afdocs/report.json new file mode 100644 index 00000000..a0a8d9cc --- /dev/null +++ b/clomonitor-core/src/testdata/afdocs/report.json @@ -0,0 +1,181 @@ +{ + "url": "https://prometheus.io", + "timestamp": "2026-09-07T00:50:57.601Z", + "specUrl": "https://agentdocsspec.com/spec/", + "results": [ + { + "id": "llms-txt-exists", + "category": "content-discoverability", + "status": "fail", + "message": "No llms.txt found at any candidate location (https://prometheus.io/llms.txt, https://prometheus.io/docs/llms.txt)" + }, + { + "id": "llms-txt-valid", + "category": "content-discoverability", + "status": "skip", + "message": "Skipped: dependency check did not pass", + "dependsOn": [ + "llms-txt-exists" + ] + }, + { + "id": "llms-txt-size", + "category": "content-discoverability", + "status": "skip", + "message": "Skipped: dependency check did not pass", + "dependsOn": [ + "llms-txt-exists" + ] + }, + { + "id": "llms-txt-links-resolve", + "category": "content-discoverability", + "status": "skip", + "message": "Skipped: dependency check did not pass", + "dependsOn": [ + "llms-txt-exists" + ] + }, + { + "id": "llms-txt-links-markdown", + "category": "content-discoverability", + "status": "skip", + "message": "Skipped: dependency check did not pass", + "dependsOn": [ + "llms-txt-exists" + ] + }, + { + "id": "llms-txt-directive-html", + "category": "content-discoverability", + "status": "fail", + "message": "No llms.txt directive found in HTML of any of 50 sampled pages" + }, + { + "id": "llms-txt-directive-md", + "category": "content-discoverability", + "status": "fail", + "message": "Could not fetch markdown for any of 50 pages; 50 had no markdown version" + }, + { + "id": "markdown-url-support", + "category": "markdown-availability", + "status": "fail", + "message": "No sampled pages support .md URLs (0/50 tested)" + }, + { + "id": "content-negotiation", + "category": "markdown-availability", + "status": "fail", + "message": "Server ignores Accept: text/markdown header (0/50 sampled pages return markdown)" + }, + { + "id": "rendering-strategy", + "category": "page-size", + "status": "pass", + "message": "All 50 sampled pages contain server-rendered content" + }, + { + "id": "page-size-markdown", + "category": "page-size", + "status": "skip", + "message": "Skipped: dependency check did not pass", + "dependsOn": [ + "markdown-url-support", + "content-negotiation" + ] + }, + { + "id": "page-size-html", + "category": "page-size", + "status": "pass", + "message": "All 50 sampled pages under 50K chars (median 160K HTML \u2192 12K markdown (92% boilerplate))" + }, + { + "id": "content-start-position", + "category": "page-size", + "status": "pass", + "message": "Content starts within first 10% on all 50 sampled pages (median 0%)" + }, + { + "id": "tabbed-content-serialization", + "category": "content-structure", + "status": "pass", + "message": "No tabbed content detected across 50 sampled pages" + }, + { + "id": "section-header-quality", + "category": "content-structure", + "status": "pass", + "message": "No tabbed content found; header quality check not applicable" + }, + { + "id": "markdown-code-fence-validity", + "category": "content-structure", + "status": "skip", + "message": "Site does not serve markdown content; nothing to analyze" + }, + { + "id": "http-status-codes", + "category": "url-stability", + "status": "pass", + "message": "All 50 sampled pages return proper error codes for bad URLs" + }, + { + "id": "redirect-behavior", + "category": "url-stability", + "status": "pass", + "message": "All 7 redirect(s) across 50 sampled pages are same-host HTTP redirects" + }, + { + "id": "llms-txt-coverage", + "category": "observability", + "status": "skip", + "message": "Skipped: dependency check did not pass", + "dependsOn": [ + "llms-txt-exists" + ] + }, + { + "id": "markdown-content-parity", + "category": "observability", + "status": "skip", + "message": "Skipped: dependency check did not pass", + "dependsOn": [ + "markdown-url-support", + "content-negotiation" + ] + }, + { + "id": "cache-header-hygiene", + "category": "observability", + "status": "warn", + "message": "50 of 50 endpoints have moderate cache lifetimes (1\u201324 hours)" + }, + { + "id": "auth-gate-detection", + "category": "authentication", + "status": "pass", + "message": "All 50 sampled pages are publicly accessible" + }, + { + "id": "auth-alternative-access", + "category": "authentication", + "status": "skip", + "message": "All docs pages are publicly accessible; no alternative access paths needed" + } + ], + "summary": { + "total": 23, + "pass": 8, + "warn": 1, + "fail": 5, + "skip": 9, + "error": 0 + }, + "discoverySources": [ + "sitemap" + ], + "testedPages": 50, + "samplingStrategy": "random" +} \ No newline at end of file diff --git a/clomonitor-linter/Dockerfile b/clomonitor-linter/Dockerfile index 8c16f6fa..f236eb74 100644 --- a/clomonitor-linter/Dockerfile +++ b/clomonitor-linter/Dockerfile @@ -25,7 +25,8 @@ RUN make build # Final stage FROM alpine:3.24.1 -RUN addgroup -S clomonitor && adduser -S clomonitor -G clomonitor +RUN apk --no-cache add nodejs npm && addgroup -S clomonitor && adduser -S clomonitor -G clomonitor +RUN npm install -g afdocs@0.20.0 USER clomonitor WORKDIR /home/clomonitor COPY --from=builder /clomonitor/target/release/clomonitor-linter /usr/local/bin diff --git a/clomonitor-linter/src/table.rs b/clomonitor-linter/src/table.rs index b725b162..327d4392 100644 --- a/clomonitor-linter/src/table.rs +++ b/clomonitor-linter/src/table.rs @@ -55,6 +55,10 @@ pub(crate) fn display( cell_entry("Documentation"), cell_score(score.documentation), ]) + .add_row(vec![ + cell_entry("Agent readiness"), + cell_score(score.agent_readiness), + ]) .add_row(vec![cell_entry("License"), cell_score(score.license)]) .add_row(vec![ cell_entry("Best practices"), @@ -111,6 +115,34 @@ pub(crate) fn display( cell_entry("Documentation / Website"), cell_check(report.documentation.website.as_ref()), ]) + .add_row(vec![ + cell_entry("Agent readiness / Authentication"), + cell_check(report.agent_readiness.authentication.as_ref()), + ]) + .add_row(vec![ + cell_entry("Agent readiness / Content discoverability"), + cell_check(report.agent_readiness.content_discoverability.as_ref()), + ]) + .add_row(vec![ + cell_entry("Agent readiness / Content structure"), + cell_check(report.agent_readiness.content_structure.as_ref()), + ]) + .add_row(vec![ + cell_entry("Agent readiness / Markdown availability"), + cell_check(report.agent_readiness.markdown_availability.as_ref()), + ]) + .add_row(vec![ + cell_entry("Agent readiness / Observability"), + cell_check(report.agent_readiness.observability.as_ref()), + ]) + .add_row(vec![ + cell_entry("Agent readiness / Page size"), + cell_check(report.agent_readiness.page_size.as_ref()), + ]) + .add_row(vec![ + cell_entry("Agent readiness / URL stability"), + cell_check(report.agent_readiness.url_stability.as_ref()), + ]) .add_row(vec![ cell_entry("License"), Cell::new(match &report.license.license_spdx_id { @@ -317,7 +349,8 @@ mod tests { use clomonitor_core::{ linter::{ - BestPractices, CheckOutput, CheckSet, Documentation, Legal, License, Report, Security, + AgentReadiness, BestPractices, CheckOutput, CheckSet, Documentation, Legal, License, + Report, Security, }, score::Score, }; @@ -342,6 +375,15 @@ mod tests { summary_table: Some(CheckOutput::passed()), website: Some(CheckOutput::passed()), }, + agent_readiness: AgentReadiness { + authentication: Some(CheckOutput::passed()), + content_discoverability: Some(CheckOutput::passed()), + content_structure: Some(CheckOutput::passed()), + markdown_availability: Some(CheckOutput::passed()), + observability: Some(CheckOutput::passed()), + page_size: Some(CheckOutput::passed()), + url_stability: Some(CheckOutput::passed()), + }, license: License { license_approved: Some(CheckOutput::passed()), license_scanning: Some( @@ -380,9 +422,11 @@ mod tests { }; let score = Score { global: 99.999_999_999_999_99, - global_weight: 5, + global_weight: 6, documentation: Some(100.0), documentation_weight: Some(1), + agent_readiness: Some(100.0), + agent_readiness_weight: Some(1), license: Some(100.0), license_weight: Some(1), best_practices: Some(100.0), diff --git a/clomonitor-linter/src/testdata/display.golden b/clomonitor-linter/src/testdata/display.golden index 82b65a8d..024f51bc 100644 --- a/clomonitor-linter/src/testdata/display.golden +++ b/clomonitor-linter/src/testdata/display.golden @@ -13,21 +13,23 @@ Repository information Score summary -╭────────────────┬───────╮ -│ Section ┆ Score │ -╞════════════════╪═══════╡ -│ Global ┆ 100 │ -├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌┤ -│ Documentation ┆ 100 │ -├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌┤ -│ License ┆ 100 │ -├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌┤ -│ Best practices ┆ 100 │ -├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌┤ -│ Security ┆ 100 │ -├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌┤ -│ Legal ┆ 100 │ -╰────────────────┴───────╯ +╭─────────────────┬───────╮ +│ Section ┆ Score │ +╞═════════════════╪═══════╡ +│ Global ┆ 100 │ +├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌┤ +│ Documentation ┆ 100 │ +├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌┤ +│ Agent readiness ┆ 100 │ +├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌┤ +│ License ┆ 100 │ +├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌┤ +│ Best practices ┆ 100 │ +├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌┤ +│ Security ┆ 100 │ +├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌┤ +│ Legal ┆ 100 │ +╰─────────────────┴───────╯ Checks summary @@ -54,6 +56,20 @@ Checks summary ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ │ Documentation / Website ┆ ✓ │ ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Agent readiness / Authentication ┆ ✓ │ +├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Agent readiness / Content discoverability ┆ ✓ │ +├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Agent readiness / Content structure ┆ ✓ │ +├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Agent readiness / Markdown availability ┆ ✓ │ +├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Agent readiness / Observability ┆ ✓ │ +├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Agent readiness / Page size ┆ ✓ │ +├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ Agent readiness / URL stability ┆ ✓ │ +├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ │ License ┆ Apache-2.0 │ ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤ │ License / Approved ┆ ✓ │ diff --git a/clomonitor-tracker/Dockerfile b/clomonitor-tracker/Dockerfile index a1c7ea64..f229dffb 100644 --- a/clomonitor-tracker/Dockerfile +++ b/clomonitor-tracker/Dockerfile @@ -25,7 +25,8 @@ RUN make build # Final stage FROM alpine:3.24.1 -RUN apk --no-cache add ca-certificates git && addgroup -S clomonitor && adduser -S clomonitor -G clomonitor +RUN apk --no-cache add ca-certificates git nodejs npm && addgroup -S clomonitor && adduser -S clomonitor -G clomonitor +RUN npm install -g afdocs@0.20.0 USER clomonitor WORKDIR /home/clomonitor COPY --from=builder /clomonitor/target/release/clomonitor-tracker /usr/local/bin diff --git a/database/migrations/functions/repositories/get_repositories_with_checks.sql b/database/migrations/functions/repositories/get_repositories_with_checks.sql index b27907e9..77dcd05b 100644 --- a/database/migrations/functions/repositories/get_repositories_with_checks.sql +++ b/database/migrations/functions/repositories/get_repositories_with_checks.sql @@ -17,6 +17,13 @@ returns setof text as $$ (rp.data->'documentation'->'roadmap'->'passed')::boolean as roadmap, (rp.data->'documentation'->'summary_table'->'passed')::boolean as summary_table, (rp.data->'documentation'->'website'->'passed')::boolean as website, + (rp.data->'agent_readiness'->'authentication'->'passed')::boolean as authentication, + (rp.data->'agent_readiness'->'content_discoverability'->'passed')::boolean as content_discoverability, + (rp.data->'agent_readiness'->'content_structure'->'passed')::boolean as content_structure, + (rp.data->'agent_readiness'->'markdown_availability'->'passed')::boolean as markdown_availability, + (rp.data->'agent_readiness'->'observability'->'passed')::boolean as observability, + (rp.data->'agent_readiness'->'page_size'->'passed')::boolean as page_size, + (rp.data->'agent_readiness'->'url_stability'->'passed')::boolean as url_stability, (rp.data->'license'->'license_approved'->'passed')::boolean as license_approved, (rp.data->'license'->'license_scanning'->'passed')::boolean as license_scanning, coalesce((rp.data->'license'->'license_spdx_id'->>'value')::text, 'Not detected') as license_spdx_id, @@ -50,7 +57,7 @@ returns setof text as $$ join report rp using (repository_id) order by p.foundation_id asc, p.name asc ) - select 'Foundation,Project,Repository URL,Check Sets,Adopters,Changelog,Code of Conduct,Contributing,Governance,Maintainers,Readme,Roadmap,Summary Table,Website,License Approved,License Scanning,License SPDX ID,Analytics,ArtifactHub Badge,CLA,Community Meeting,DCO,GitHub discussions,OpenSSF best practices badge,OpenSSF Scorecard badge,Recent Release,Slack Presence,Binary Artifacts,Code Review,Dangerous Workflow,Dependencies Policy,Dependency Update Tool,Maintained,SBOM,Security Insights,Security Policy,Signed Releases,Token Permissions,Trademark Disclaimer' + select 'Foundation,Project,Repository URL,Check Sets,Adopters,Changelog,Code of Conduct,Contributing,Governance,Maintainers,Readme,Roadmap,Summary Table,Website,Authentication,Content Discoverability,Content Structure,Markdown Availability,Observability,Page Size,URL Stability,License Approved,License Scanning,License SPDX ID,Analytics,ArtifactHub Badge,CLA,Community Meeting,DCO,GitHub discussions,OpenSSF best practices badge,OpenSSF Scorecard badge,Recent Release,Slack Presence,Binary Artifacts,Code Review,Dangerous Workflow,Dependencies Policy,Dependency Update Tool,Maintained,SBOM,Security Insights,Security Policy,Signed Releases,Token Permissions,Trademark Disclaimer' union all select rtrim(ltrim(r.*::text, '('), ')') from repositories r; $$ language sql; diff --git a/database/migrations/functions/stats/get_stats.sql b/database/migrations/functions/stats/get_stats.sql index d142212e..fe3bf2d6 100644 --- a/database/migrations/functions/stats/get_stats.sql +++ b/database/migrations/functions/stats/get_stats.sql @@ -98,6 +98,7 @@ returns json as $$ ( select jsonb_build_object( 'documentation', (average_section_score(p_foundation, 'documentation', null)), + 'agent_readiness', (average_section_score(p_foundation, 'agent_readiness', null)), 'license', (average_section_score(p_foundation, 'license', null)), 'best_practices', (average_section_score(p_foundation, 'best_practices', null)), 'security', (average_section_score(p_foundation, 'security', null)), @@ -113,6 +114,7 @@ returns json as $$ ( select jsonb_build_object( 'documentation', (average_section_score(p_foundation, 'documentation', maturity)), + 'agent_readiness', (average_section_score(p_foundation, 'agent_readiness', maturity)), 'license', (average_section_score(p_foundation, 'license', maturity)), 'best_practices', (average_section_score(p_foundation, 'best_practices', maturity)), 'security', (average_section_score(p_foundation, 'security', maturity)), @@ -163,6 +165,15 @@ returns json as $$ 'summary_table', repositories_passing_check(p_foundation, 'documentation', 'summary_table'), 'website', repositories_passing_check(p_foundation, 'documentation', 'website') ), + 'agent_readiness', json_build_object( + 'authentication', repositories_passing_check(p_foundation, 'agent_readiness', 'authentication'), + 'content_discoverability', repositories_passing_check(p_foundation, 'agent_readiness', 'content_discoverability'), + 'content_structure', repositories_passing_check(p_foundation, 'agent_readiness', 'content_structure'), + 'markdown_availability', repositories_passing_check(p_foundation, 'agent_readiness', 'markdown_availability'), + 'observability', repositories_passing_check(p_foundation, 'agent_readiness', 'observability'), + 'page_size', repositories_passing_check(p_foundation, 'agent_readiness', 'page_size'), + 'url_stability', repositories_passing_check(p_foundation, 'agent_readiness', 'url_stability') + ), 'license', json_build_object( 'license_approved', repositories_passing_check(p_foundation, 'license', 'license_approved'), 'license_scanning', repositories_passing_check(p_foundation, 'license', 'license_scanning'), diff --git a/database/tests/functions/repositories/get_repositories_with_checks.sql b/database/tests/functions/repositories/get_repositories_with_checks.sql index 1932a56e..b4a83c02 100644 --- a/database/tests/functions/repositories/get_repositories_with_checks.sql +++ b/database/tests/functions/repositories/get_repositories_with_checks.sql @@ -57,6 +57,30 @@ insert into report ( ) values ( '00000000-0000-0000-0001-000000000000', '{ + "agent_readiness": { + "authentication": { + "passed": true + }, + "content_discoverability": { + "url": "https://www.fluentd.org/llms.txt", + "passed": true + }, + "content_structure": { + "passed": false + }, + "markdown_availability": { + "passed": true + }, + "observability": { + "passed": false + }, + "page_size": { + "passed": true + }, + "url_stability": { + "passed": true + } + }, "legal": { "trademark_disclaimer": { "passed": false @@ -227,9 +251,9 @@ select results_eq( $$, $$ values - ('Foundation,Project,Repository URL,Check Sets,Adopters,Changelog,Code of Conduct,Contributing,Governance,Maintainers,Readme,Roadmap,Summary Table,Website,License Approved,License Scanning,License SPDX ID,Analytics,ArtifactHub Badge,CLA,Community Meeting,DCO,GitHub discussions,OpenSSF best practices badge,OpenSSF Scorecard badge,Recent Release,Slack Presence,Binary Artifacts,Code Review,Dangerous Workflow,Dependencies Policy,Dependency Update Tool,Maintained,SBOM,Security Insights,Security Policy,Signed Releases,Token Permissions,Trademark Disclaimer'), - ('cncf,project1,https://repo1.url,"{code,community}",t,t,t,t,t,t,t,f,f,t,t,f,Apache-2.0,GA4,f,t,f,t,t,t,t,t,f,t,t,t,t,f,t,f,t,t,f,f,f'), - ('cncf,project1,https://repo2.url,{docs},,,,,,,f,,,,t,,Apache-2.0,,,,,,,,,,,,,,,,,,,,,,') + ('Foundation,Project,Repository URL,Check Sets,Adopters,Changelog,Code of Conduct,Contributing,Governance,Maintainers,Readme,Roadmap,Summary Table,Website,Authentication,Content Discoverability,Content Structure,Markdown Availability,Observability,Page Size,URL Stability,License Approved,License Scanning,License SPDX ID,Analytics,ArtifactHub Badge,CLA,Community Meeting,DCO,GitHub discussions,OpenSSF best practices badge,OpenSSF Scorecard badge,Recent Release,Slack Presence,Binary Artifacts,Code Review,Dangerous Workflow,Dependencies Policy,Dependency Update Tool,Maintained,SBOM,Security Insights,Security Policy,Signed Releases,Token Permissions,Trademark Disclaimer'), + ('cncf,project1,https://repo1.url,"{code,community}",t,t,t,t,t,t,t,f,f,t,t,t,f,t,f,t,t,t,f,Apache-2.0,GA4,f,t,f,t,t,t,t,t,f,t,t,t,t,f,t,f,t,t,f,f,f'), + ('cncf,project1,https://repo2.url,{docs},,,,,,,f,,,,,,,,,,,t,,Apache-2.0,,,,,,,,,,,,,,,,,,,,,,') $$, 'Return all repositories with all checks' ); diff --git a/database/tests/functions/stats/get_stats.sql b/database/tests/functions/stats/get_stats.sql index 950cf16d..0dd4f20f 100644 --- a/database/tests/functions/stats/get_stats.sql +++ b/database/tests/functions/stats/get_stats.sql @@ -17,7 +17,7 @@ insert into project ( '00000000-0001-0000-0000-000000000000', 'project1', 'category1', - '{"global": 95.0, "license": 100.0, "security": 100.0, "documentation": 80.0, "best_practices": 100.0}', + '{"global": 95.0, "license": 100.0, "security": 100.0, "documentation": 80.0, "agent_readiness": 90.0, "best_practices": 100.0}', 'a', '2022-02-25', 'sandbox', @@ -36,7 +36,7 @@ insert into project ( '00000000-0002-0000-0000-000000000000', 'project2', 'category1', - '{"global": 70.0, "license": 80.0, "security": 60.0, "documentation": 70.0, "best_practices": 70.0}', + '{"global": 70.0, "license": 80.0, "security": 60.0, "documentation": 70.0, "agent_readiness": 60.0, "best_practices": 70.0}', 'b', '2021-02-24', 'graduated', @@ -108,6 +108,30 @@ insert into report ( ) values ( '00000000-0000-0000-0001-000000000000', '{ + "agent_readiness": { + "authentication": { + "passed": true + }, + "content_discoverability": { + "url": "https://www.fluentd.org/llms.txt", + "passed": true + }, + "content_structure": { + "passed": false + }, + "markdown_availability": { + "passed": true + }, + "observability": { + "passed": false + }, + "page_size": { + "passed": true + }, + "url_stability": { + "passed": true + } + }, "legal": { "trademark_disclaimer": { "passed": false @@ -249,6 +273,29 @@ insert into report ( ) values ( '00000000-0000-0000-0002-000000000000', '{ + "agent_readiness": { + "authentication": { + "passed": true + }, + "content_discoverability": { + "passed": false + }, + "content_structure": { + "passed": false + }, + "markdown_availability": { + "passed": false + }, + "observability": { + "passed": false + }, + "page_size": { + "passed": true + }, + "url_stability": { + "passed": true + } + }, "legal": { "trademark_disclaimer": { "passed": false @@ -564,18 +611,21 @@ select is( "license": 77, "security": 73, "documentation": 73, + "agent_readiness": 75, "best_practices": 70 }, "graduated": { "license": 65, "security": 60, "documentation": 70, + "agent_readiness": 60, "best_practices": 55 }, "sandbox": { "license": 100, "security": 100, "documentation": 80, + "agent_readiness": 90, "best_practices": 100 } } @@ -594,6 +644,15 @@ select is( "summary_table": 0, "website": 67 }, + "agent_readiness": { + "authentication": 100, + "content_discoverability": 50, + "content_structure": 0, + "markdown_availability": 50, + "observability": 0, + "page_size": 100, + "url_stability": 100 + }, "license": { "license_approved": 67, "license_scanning": 0, diff --git a/docs/checks.md b/docs/checks.md index 8ca7458d..f87efdaf 100644 --- a/docs/checks.md +++ b/docs/checks.md @@ -51,6 +51,13 @@ Checks are organized in `check sets`. Each `check set` defines a number of check - Documentation / Roadmap - Documentation / Summary table - Documentation / Website + - Agent readiness / Authentication + - Agent readiness / Content discoverability + - Agent readiness / Content structure + - Agent readiness / Markdown availability + - Agent readiness / Observability + - Agent readiness / Page size + - Agent readiness / URL stability - Best practices / Analytics - Best practices / Community meeting - Best practices / GitHub discussions @@ -61,6 +68,13 @@ Checks are organized in `check sets`. Each `check set` defines a number of check - **docs** (recommended for other documentation repositories) - Documentation / Readme + - Agent readiness / Authentication + - Agent readiness / Content discoverability + - Agent readiness / Content structure + - Agent readiness / Markdown availability + - Agent readiness / Observability + - Agent readiness / Page size + - Agent readiness / URL stability - License - License / Approved @@ -315,6 +329,68 @@ This check passes if: - A website *url* is configured in the Github repository. +## Agent readiness + +The checks in this category evaluate how well AI agents can discover and consume the project's documentation. CLOMonitor delegates them to [AFDocs](https://afdocs.dev), the reference implementation of the [Agent-Friendly Documentation Spec](https://agentdocsspec.com), which is run on the project's *website* (the one configured in the GitHub repository). When the repository has no website configured, this category is not applicable and won't be displayed. + +Each check corresponds to an AFDocs category and passes when none of the checks in it fail (warnings are tolerated). When AFDocs skips all the checks in a category, the check is marked as exempt. + +### Authentication + +**ID**: `authentication` + +The project's documentation should be accessible to AI agents without authentication gates or paywalls. + +*This is an AFDocs check. For more details, please see the [authentication and access](https://agentdocsspec.com/spec/#category-7-authentication-and-access) category documentation in the Agent-Friendly Documentation Spec.* + +### Content discoverability + +**ID**: `content_discoverability` + +The project's website should provide a valid [llms.txt](https://llmstxt.org) index so AI agents can discover its documentation. + +*This is an AFDocs check. For more details, please see the [content discoverability](https://agentdocsspec.com/spec/#category-1-content-discoverability) category documentation in the Agent-Friendly Documentation Spec.* + +### Content structure + +**ID**: `content_structure` + +The project's documentation content should be well structured, with quality section headers and valid code blocks. + +*This is an AFDocs check. For more details, please see the [content structure](https://agentdocsspec.com/spec/#category-4-content-structure) category documentation in the Agent-Friendly Documentation Spec.* + +### Markdown availability + +**ID**: `markdown_availability` + +The project's documentation pages should be available as Markdown for AI agents to consume. + +*This is an AFDocs check. For more details, please see the [markdown availability](https://agentdocsspec.com/spec/#category-2-markdown-availability) category documentation in the Agent-Friendly Documentation Spec.* + +### Observability + +**ID**: `observability` + +The project's website llms.txt index should cover its documentation pages, and their Markdown version should match the HTML one. + +*This is an AFDocs check. For more details, please see the [observability and content health](https://agentdocsspec.com/spec/#category-6-observability-and-content-health) category documentation in the Agent-Friendly Documentation Spec.* + +### Page size + +**ID**: `page_size` + +The project's documentation pages should be reasonably sized and not require JavaScript to render their content. + +*This is an AFDocs check. For more details, please see the [page size and truncation risk](https://agentdocsspec.com/spec/#category-3-page-size-and-truncation-risk) category documentation in the Agent-Friendly Documentation Spec.* + +### URL stability + +**ID**: `url_stability` + +The project's documentation URLs should be stable and use proper HTTP status codes and redirects. + +*This is an AFDocs check. For more details, please see the [URL stability and redirects](https://agentdocsspec.com/spec/#category-5-url-stability-and-redirects) category documentation in the Agent-Friendly Documentation Spec.* + ## License ### SPDX id diff --git a/docs/dev.md b/docs/dev.md index f1c7e5ef..dadd78c3 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -183,7 +183,7 @@ Depending on the speed of your Internet connection and machine, this may take on In the section above, we saw how the `tracker` is able to lint all repositories registered in the database. But sometimes, it may be desirable to lint a single repository manually in an isolated way, maybe to quickly test some checks or to integrate with some other processes, like continuous integration or deployment tools. The `linter CLI` tool is designed to help in those scenarios. -CLOMonitor delegates some of the security checks to [OpenSSF Scorecard](https://github.com/ossf/scorecard), so you'll need to [install it](https://github.com/ossf/scorecard#installation) before running `clomonitor-linter` locally. Both CLOMonitor and [OpenSSF Scorecard](https://github.com/ossf/scorecard) use the Github GraphQL API for some checks, which requires authentication. A Github token (with `public_repo` scope) **must** be provided via the `GITHUB_TOKEN` environment variable to authenticate those requests. +CLOMonitor delegates some of the security checks to [OpenSSF Scorecard](https://github.com/ossf/scorecard), so you'll need to [install it](https://github.com/ossf/scorecard#installation) before running `clomonitor-linter` locally. Similarly, the agent readiness checks are delegated to [AFDocs](https://afdocs.dev), which you can install from [npm](https://www.npmjs.com/package/afdocs) (`npm install -g afdocs`). Both CLOMonitor and [OpenSSF Scorecard](https://github.com/ossf/scorecard) use the Github GraphQL API for some checks, which requires authentication. A Github token (with `public_repo` scope) **must** be provided via the `GITHUB_TOKEN` environment variable to authenticate those requests. If you are using the aliases provided below, you can run it this way: diff --git a/web/src/data.tsx b/web/src/data.tsx index 24f75c50..78eea14b 100644 --- a/web/src/data.tsx +++ b/web/src/data.tsx @@ -2,7 +2,20 @@ import { ExternalLink } from 'clo-ui/components/ExternalLink'; import { Foundation } from 'clo-ui/components/Foundation'; import { Maturity } from 'clo-ui/components/Maturity'; import { SampleQuery } from 'clo-ui/components/SampleQueries'; -import { BiLock, BiMedal, BiShieldQuarter, BiTable, BiTrophy, BiWorld } from 'react-icons/bi'; +import { + BiCompass, + BiFile, + BiKey, + BiLink, + BiLock, + BiMedal, + BiPulse, + BiShieldQuarter, + BiSitemap, + BiTable, + BiTrophy, + BiWorld, +} from 'react-icons/bi'; import { BsCalendar3, BsUiChecks } from 'react-icons/bs'; import { CgFileDocument, CgReadme } from 'react-icons/cg'; import { @@ -12,6 +25,7 @@ import { FaExclamationTriangle, FaFileContract, FaFileSignature, + FaMarkdown, FaRobot, FaSignature, FaSlack, @@ -28,7 +42,7 @@ import { HiOutlinePencilAlt, HiTerminal } from 'react-icons/hi'; import { ImOffice } from 'react-icons/im'; import { IoIosPeople, IoMdRibbon } from 'react-icons/io'; import { MdOutlineInventory, MdPreview } from 'react-icons/md'; -import { RiRoadMapLine, RiShieldStarLine } from 'react-icons/ri'; +import { RiRoadMapLine, RiRobot2Line, RiShieldStarLine } from 'react-icons/ri'; import QualityDot from './layout/common/QualityDot'; import { @@ -236,6 +250,7 @@ export const QUERIES: SampleQuery[] = [ ]; export const CATEGORY_ICONS = { + [ScoreType.AgentReadiness]: , [ScoreType.BestPractices]: , [ScoreType.Documentation]: , [ScoreType.Global]: , @@ -245,6 +260,7 @@ export const CATEGORY_ICONS = { }; export const CATEGORY_NAMES = { + [ScoreType.AgentReadiness]: 'Agent Readiness', [ScoreType.BestPractices]: 'Best Practices', [ScoreType.Documentation]: 'Documentation', [ScoreType.Global]: 'Global', @@ -294,6 +310,16 @@ export const REPORT_OPTIONS: ReportOptionInfo = { ), reference: '/docs/topics/checks/#artifact-hub-badge', }, + [ReportOption.Authentication]: { + icon: , + name: 'Authentication', + legend: ( + + The project's documentation should be accessible to AI agents without authentication gates or paywalls + + ), + reference: '/docs/topics/checks/#authentication', + }, [ReportOption.BinaryArtifacts]: { icon: , name: 'Binary artifacts', @@ -340,6 +366,31 @@ export const REPORT_OPTIONS: ReportOptionInfo = { ), reference: '/docs/topics/checks/#community-meeting', }, + [ReportOption.ContentDiscoverability]: { + icon: , + name: 'Content discoverability', + legend: ( + + The project's website should provide a valid{' '} + + llms.txt + {' '} + index so AI agents can discover its documentation + + ), + reference: '/docs/topics/checks/#content-discoverability', + }, + [ReportOption.ContentStructure]: { + icon: , + name: 'Content structure', + legend: ( + + The project's documentation content should be well structured, with quality section headers and valid code + blocks + + ), + reference: '/docs/topics/checks/#content-structure', + }, [ReportOption.Contributing]: { icon: , name: 'Contributing', @@ -418,6 +469,23 @@ export const REPORT_OPTIONS: ReportOptionInfo = { ), reference: '/docs/topics/checks/#maintainers', }, + [ReportOption.MarkdownAvailability]: { + icon: , + name: 'Markdown availability', + legend: The project's documentation pages should be available as Markdown for AI agents to consume, + reference: '/docs/topics/checks/#markdown-availability', + }, + [ReportOption.Observability]: { + icon: , + name: 'Observability', + legend: ( + + The project's website llms.txt index should cover its documentation pages, and their Markdown version should + match the HTML one + + ), + reference: '/docs/topics/checks/#observability', + }, [ReportOption.OpenSSFBadge]: { icon: , name: 'OpenSSF best practices badge', @@ -439,6 +507,16 @@ export const REPORT_OPTIONS: ReportOptionInfo = { ), reference: '/docs/topics/checks/#openssf-scorecard-badge', }, + [ReportOption.PageSize]: { + icon: , + name: 'Page size', + legend: ( + + The project's documentation pages should be reasonably sized and not require JavaScript to render their content + + ), + reference: '/docs/topics/checks/#page-size', + }, [ReportOption.Readme]: { icon: , name: 'Readme', @@ -545,6 +623,14 @@ export const REPORT_OPTIONS: ReportOptionInfo = { legend: Projects sites should have the Linux Foundation trademark disclaimer, reference: '/docs/topics/checks/#trademark-disclaimer', }, + [ReportOption.UrlStability]: { + icon: , + name: 'URL stability', + legend: ( + The project's documentation URLs should be stable and use proper HTTP status codes and redirects + ), + reference: '/docs/topics/checks/#url-stability', + }, [ReportOption.Website]: { icon: , name: 'Website', @@ -572,6 +658,15 @@ export const CHECKS_PER_CATEGORY: ChecksPerCategory = { ReportOption.SummaryTable, ReportOption.Website, ], + [ScoreType.AgentReadiness]: [ + ReportOption.Authentication, + ReportOption.ContentDiscoverability, + ReportOption.ContentStructure, + ReportOption.MarkdownAvailability, + ReportOption.Observability, + ReportOption.PageSize, + ReportOption.UrlStability, + ], [ScoreType.License]: [ReportOption.SPDX, ReportOption.ApprovedLicense, ReportOption.LicenseScanning], [ScoreType.BestPractices]: [ ReportOption.Analytics, diff --git a/web/src/layout/common/CategoriesSummary.tsx b/web/src/layout/common/CategoriesSummary.tsx index e96e4db0..98daca1f 100644 --- a/web/src/layout/common/CategoriesSummary.tsx +++ b/web/src/layout/common/CategoriesSummary.tsx @@ -55,6 +55,14 @@ const CategoriesSummary = (props: Props) => { linkTo={activeLink ? `${props.repoName}_${ScoreType.Documentation}` : undefined} scrollIntoView={activeLink ? props.scrollIntoView : undefined} /> + creates snapshot 1`] = ` +
+
+
+ + + + + + + Agent Readiness + +
+
+ n/a +
+
+
+
diff --git a/web/src/layout/detail/repositories/Summary.test.tsx b/web/src/layout/detail/repositories/Summary.test.tsx index d507d13e..39d5a9d6 100644 --- a/web/src/layout/detail/repositories/Summary.test.tsx +++ b/web/src/layout/detail/repositories/Summary.test.tsx @@ -81,7 +81,7 @@ describe('Summary', () => { expect(screen.getAllByText('100')).toHaveLength(5); expect(screen.getAllByText('85')).toHaveLength(3); expect(screen.getAllByText('70')).toHaveLength(3); - expect(screen.getAllByText('n/a')).toHaveLength(16); + expect(screen.getAllByText('n/a')).toHaveLength(22); }); }); @@ -111,7 +111,7 @@ describe('Summary', () => { expect(screen.getAllByText('90')).toHaveLength(2); expect(screen.getAllByText('87')).toHaveLength(2); - expect(screen.getAllByText('n/a')).toHaveLength(6); + expect(screen.getAllByText('n/a')).toHaveLength(8); }); it('renders component', async () => { diff --git a/web/src/layout/detail/repositories/Summary.tsx b/web/src/layout/detail/repositories/Summary.tsx index 51a3c703..8b66e53e 100644 --- a/web/src/layout/detail/repositories/Summary.tsx +++ b/web/src/layout/detail/repositories/Summary.tsx @@ -50,6 +50,10 @@ const Summary = (props: Props) => { {CATEGORY_ICONS[ScoreType.Documentation]} Documentation + + {CATEGORY_ICONS[ScoreType.AgentReadiness]} + Agent Readiness + {CATEGORY_ICONS[ScoreType.License]} License @@ -98,6 +102,11 @@ const Summary = (props: Props) => { onClick={() => goToAnchor(`${repo.name}_${ScoreType.Documentation}`)} /> + goToAnchor(`${repo.name}_${ScoreType.AgentReadiness}`)} + /> + goToAnchor(`${repo.name}_${ScoreType.License}`)} diff --git a/web/src/layout/detail/repositories/__snapshots__/Summary.test.tsx.snap b/web/src/layout/detail/repositories/__snapshots__/Summary.test.tsx.snap index ce7a47ce..e8f37fdf 100644 --- a/web/src/layout/detail/repositories/__snapshots__/Summary.test.tsx.snap +++ b/web/src/layout/detail/repositories/__snapshots__/Summary.test.tsx.snap @@ -93,6 +93,33 @@ exports[`Summary > creates snapshot 1`] = ` Documentation + + + + + + + + Agent Readiness + + creates snapshot 1`] = ` 90
+ +
+ + n/a + +
+ creates snapshot 1`] = ` 90
+ +
+ + n/a + +
+ creates snapshot 1`] = ` 70 + +
+ + n/a + +
+ creates snapshot 1`] = ` 70 + +
+ + n/a + +
+ creates snapshot 1`] = ` 60 + +
+ + n/a + +
+ creates snapshot 1`] = ` 70 + +
+ + n/a + +
+ creates snapshot 1`] = ` Documentation + + + + + + + + Agent Readiness + + creates snapshot 1`] = ` 60 + +
+ + n/a + +
+ creates snapshot 1`] = ` 90 + +
+ + n/a + +
+ creates snapshot 1`] = ` 90 + +
+ + n/a + +
+ creates snapshot 1`] = ` 70 + +
+ + n/a + +
+ creates snapshot 1`] = ` 70 + +
+ + n/a + +
+ creates snapshot 1`] = ` 70 + +
+ + n/a + +
+ { } getAnchorLink={getAnchorLink} /> + creates snapshot 1`] = ` +
+
+
+ + + + + + + Agent Readiness + +
+
+ n/a +
+
+
+
diff --git a/web/src/layout/search/__snapshots__/index.test.tsx.snap b/web/src/layout/search/__snapshots__/index.test.tsx.snap index 8c25ffff..4369c619 100644 --- a/web/src/layout/search/__snapshots__/index.test.tsx.snap +++ b/web/src/layout/search/__snapshots__/index.test.tsx.snap @@ -587,6 +587,48 @@ exports[`Project detail index > creates snapshot 1`] = `
+
+
+
+ + + + + + + Agent Readiness + +
+
+ n/a +
+
+
+
@@ -1126,6 +1168,48 @@ exports[`Project detail index > creates snapshot 1`] = `
+
+
+
+ + + + + + + Agent Readiness + +
+
+ n/a +
+
+
+
@@ -1665,6 +1749,48 @@ exports[`Project detail index > creates snapshot 1`] = `
+
+
+
+ + + + + + + Agent Readiness + +
+
+ n/a +
+
+
+
@@ -2204,6 +2330,48 @@ exports[`Project detail index > creates snapshot 1`] = `
+
+
+
+ + + + + + + Agent Readiness + +
+
+ n/a +
+
+
+
diff --git a/web/src/layout/stats/Average.test.tsx b/web/src/layout/stats/Average.test.tsx index 5ca01d4e..0a9ff46d 100644 --- a/web/src/layout/stats/Average.test.tsx +++ b/web/src/layout/stats/Average.test.tsx @@ -4,7 +4,15 @@ import { vi } from 'vitest'; import Average from './Average'; const defaultProps = { - data: { documentation: 85, license: 79, best_practices: 57, security: 53, legal: 40, global: 69 }, + data: { + documentation: 85, + agent_readiness: 48, + license: 79, + best_practices: 57, + security: 53, + legal: 40, + global: 69, + }, title: 'Sandbox', }; @@ -26,6 +34,8 @@ describe('Average', () => { expect(screen.getByText('Sandbox')).toBeInTheDocument(); expect(screen.getByText('Documentation')).toBeInTheDocument(); expect(screen.getByText('85%')).toBeInTheDocument(); + expect(screen.getByText('Agent Readiness')).toBeInTheDocument(); + expect(screen.getByText('48%')).toBeInTheDocument(); expect(screen.getByText('License')).toBeInTheDocument(); expect(screen.getByText('79%')).toBeInTheDocument(); expect(screen.getByText('Best Practices')).toBeInTheDocument(); @@ -35,7 +45,7 @@ describe('Average', () => { expect(screen.getByText('Legal')).toBeInTheDocument(); expect(screen.getByText('40%')).toBeInTheDocument(); - expect(screen.getAllByRole('progressbar')).toHaveLength(5); + expect(screen.getAllByRole('progressbar')).toHaveLength(6); }); }); }); diff --git a/web/src/layout/stats/Average.tsx b/web/src/layout/stats/Average.tsx index 93585359..59a37c68 100644 --- a/web/src/layout/stats/Average.tsx +++ b/web/src/layout/stats/Average.tsx @@ -1,3 +1,5 @@ +import { isUndefined } from 'lodash'; + import { CATEGORY_ICONS } from '../../data'; import { ScoreType } from '../../types'; import styles from './Average.module.css'; @@ -18,6 +20,13 @@ const Average = (props: Props) => { icon={CATEGORY_ICONS[ScoreType.Documentation]} value={props.data.documentation} /> + {!isUndefined(props.data.agent_readiness) && ( + + )} creates snapshot 1`] = `
+
+
+
+ + + +
+
+ Agent Readiness +
+
+
+
+
+
+
+
+
+ + 48% + +
+
+
diff --git a/web/src/layout/stats/index.tsx b/web/src/layout/stats/index.tsx index 11343166..8f0fd46c 100644 --- a/web/src/layout/stats/index.tsx +++ b/web/src/layout/stats/index.tsx @@ -982,6 +982,13 @@ const StatsView = () => { data={stats.repositories.passing_check.documentation} onSelectCheck={selectCheck} /> + {!isUndefined(stats.repositories.passing_check.agent_readiness) && ( + + )}