From 9e071453b60538d30a54434e4202bc34e3a8ca11 Mon Sep 17 00:00:00 2001 From: apiraino Date: Wed, 10 Jun 2026 19:25:27 +0200 Subject: [PATCH] Add T-compiler P-high issues triage module --- Cargo.lock | 26 ++ Cargo.toml | 2 + utils/compiler-p-high/Cargo.toml | 30 ++ utils/compiler-p-high/src/actions.rs | 149 +++++++ utils/compiler-p-high/src/github.rs | 9 + utils/compiler-p-high/src/github/client.rs | 328 ++++++++++++++ utils/compiler-p-high/src/github/issue.rs | 267 ++++++++++++ .../compiler-p-high/src/github/issue_query.rs | 75 ++++ utils/compiler-p-high/src/github/repos.rs | 402 ++++++++++++++++++ utils/compiler-p-high/src/github/utils.rs | 11 + utils/compiler-p-high/src/lib.rs | 6 + utils/compiler-p-high/src/main.rs | 109 +++++ utils/compiler-p-high/src/team_data.rs | 198 +++++++++ utils/compiler-p-high/templates/_issue.tt | 11 + utils/compiler-p-high/templates/_issues.tt | 9 + .../templates/p_high_issues.tt | 24 ++ 16 files changed, 1656 insertions(+) create mode 100644 utils/compiler-p-high/Cargo.toml create mode 100644 utils/compiler-p-high/src/actions.rs create mode 100644 utils/compiler-p-high/src/github.rs create mode 100644 utils/compiler-p-high/src/github/client.rs create mode 100644 utils/compiler-p-high/src/github/issue.rs create mode 100644 utils/compiler-p-high/src/github/issue_query.rs create mode 100644 utils/compiler-p-high/src/github/repos.rs create mode 100644 utils/compiler-p-high/src/github/utils.rs create mode 100644 utils/compiler-p-high/src/lib.rs create mode 100644 utils/compiler-p-high/src/main.rs create mode 100644 utils/compiler-p-high/src/team_data.rs create mode 100644 utils/compiler-p-high/templates/_issue.tt create mode 100644 utils/compiler-p-high/templates/_issues.tt create mode 100644 utils/compiler-p-high/templates/p_high_issues.tt diff --git a/Cargo.lock b/Cargo.lock index 06d00f60f..de595dced 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -580,6 +580,32 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "compiler-p-high" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "futures", + "http", + "http-body-util", + "itertools", + "octocrab", + "regex", + "reqwest", + "rust_team_data", + "secrecy", + "serde", + "serde_json", + "tera", + "tokio", + "tracing", + "tracing-subscriber", + "url", +] + [[package]] name = "comrak" version = "0.38.0" diff --git a/Cargo.toml b/Cargo.toml index d4ff70ebf..9ad0a1182 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,8 @@ authors = ["Mark Rousskov "] edition = "2024" [workspace] +resolver = "3" +members = ["utils/compiler-p-high"] [dependencies] serde_json = "1" diff --git a/utils/compiler-p-high/Cargo.toml b/utils/compiler-p-high/Cargo.toml new file mode 100644 index 000000000..f827558f6 --- /dev/null +++ b/utils/compiler-p-high/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "compiler-p-high" +version = "0.1.0" +edition = "2024" + +[dependencies] +anyhow = "1.0.98" +async-trait = "0.1.31" +chrono = { version = "0.4.38", default-features = false, features = ["serde"] } +octocrab = { version = "0.44" } +tera = { version = "1.3.1", default-features = false, features = ["builtins"] } +tokio = { version = "1", default-features = false, features = ["macros", "time", "rt"] } +tracing-subscriber = { version = "0.3", default-features = false, features = ["env-filter", "fmt"] } + +# Rust team data and its deps +rust_team_data = { git = "https://github.com/rust-lang/team" } +reqwest = { version = "0.12", features = ["json", "blocking"] } +serde = { version = "1", default-features = false, features = ["derive"] } +tracing = "0.1" + +# github.rs and its deps +serde_json = "1" +regex = "1" +bytes = "1.10.1" +itertools = "0.14.0" +secrecy = { version = "0.10", default-features = false, features = ["serde"] } +http = "1.4.0" +http-body-util = "0.1.3" +futures = { version = "0.3", default-features = false, features = ["std"] } +url = "2.1.0" diff --git a/utils/compiler-p-high/src/actions.rs b/utils/compiler-p-high/src/actions.rs new file mode 100644 index 000000000..46cf2a3d2 --- /dev/null +++ b/utils/compiler-p-high/src/actions.rs @@ -0,0 +1,149 @@ +use chrono::{DateTime, Utc}; +use octocrab::Octocrab; +use std::collections::HashMap; +use std::sync::{Arc, LazyLock}; + +use async_trait::async_trait; +use tera::{Context, Tera}; + +// TODO: can this crate be replaced with octocrab? +use crate::github::{self, GithubClient, Repository}; +use crate::team_data::TeamClient; + +#[async_trait] +pub trait Action { + async fn call(&self, octocrab: &Octocrab) -> anyhow::Result; +} + +pub struct Step<'a> { + pub name: &'a str, + pub actions: Vec>, +} + +pub struct Query<'a> { + /// Vec of (owner, name) + pub repos: Vec<(&'a str, &'a str)>, + pub queries: Vec>, +} + +#[derive(Copy, Clone)] +pub enum QueryKind { + List, + Count, +} + +pub struct QueryMap<'a> { + pub name: &'a str, + pub kind: QueryKind, + pub query: Arc, +} + +#[derive(Debug, serde::Serialize)] +pub struct IssueDecorator { + pub number: u64, + pub title: String, + pub html_url: String, + pub repo_name: String, + pub labels: String, + pub author: String, + pub team: String, + pub assignees: String, + // Human (readable) timestamps + pub created_at_hts: String, + pub updated_at_hts: String, + pub is_blocked: bool, +} + +// TODO: embed templates in binary +// (so the compiled binary can run from everywhere) +pub static TEMPLATES: LazyLock = LazyLock::new(|| match Tera::new("templates/*") { + Ok(t) => t, + Err(e) => { + println!("Parsing error(s): {e}"); + ::std::process::exit(1); + } +}); + +pub fn to_human(d: DateTime) -> String { + let d1 = chrono::Utc::now() - d; + let days = d1.num_days(); + if days > 60 { + format!("{} months ago", days / 30) + } else { + format!("about {days} days ago") + } +} + +// TODO: try using the octocrab client +#[async_trait] +impl Action for Step<'_> { + async fn call(&self, _octocrab: &Octocrab) -> anyhow::Result { + let mut gh = GithubClient::new_from_env(); + gh.set_retry_rate_limit(true); + let team_api = TeamClient::new_from_env(); + + let mut context = Context::new(); + let mut results = HashMap::new(); + + let mut handles: Vec)>>> = + Vec::new(); + let semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(5)); + + for Query { repos, queries } in &self.actions { + for repo in repos { + let repository = Repository { + full_name: format!("{}/{}", repo.0, repo.1), + // These are unused for query. + default_branch: "main".to_string(), + fork: false, + parent: None, + }; + + for QueryMap { name, kind, query } in queries { + let semaphore = semaphore.clone(); + let name = String::from(*name); + let kind = *kind; + let repository = repository.clone(); + let gh = gh.clone(); + let team_api = team_api.clone(); + let query = query.clone(); + handles.push(tokio::task::spawn(async move { + let _permit = semaphore.acquire().await?; + let issues = query.query(&repository, &gh, &team_api).await?; + Ok((name, kind, issues)) + })); + } + } + } + + for handle in handles { + let (name, kind, issues) = handle.await.unwrap()?; + match kind { + QueryKind::List => { + results.entry(name).or_insert(Vec::new()).extend(issues); + } + QueryKind::Count => { + let count = issues.len(); + let result = if let Some(value) = context.get(&name) { + value.as_u64().unwrap() + count as u64 + } else { + count as u64 + }; + + context.insert(name, &result); + } + } + } + + for (name, issues) in &results { + context.insert(name, issues); + } + + let date = chrono::Utc::now(); + context.insert("CURRENT_DATE", &date); + + Ok(TEMPLATES + .render(&format!("{}.tt", self.name), &context) + .unwrap()) + } +} diff --git a/utils/compiler-p-high/src/github.rs b/utils/compiler-p-high/src/github.rs new file mode 100644 index 000000000..9b190a876 --- /dev/null +++ b/utils/compiler-p-high/src/github.rs @@ -0,0 +1,9 @@ +mod client; +mod issue; +pub mod issue_query; +mod repos; +mod utils; + +pub use client::{GithubClient, default_token_from_env}; +pub use issue::*; +pub use repos::*; diff --git a/utils/compiler-p-high/src/github/client.rs b/utils/compiler-p-high/src/github/client.rs new file mode 100644 index 000000000..7b8c6b290 --- /dev/null +++ b/utils/compiler-p-high/src/github/client.rs @@ -0,0 +1,328 @@ +use anyhow::Context; +use bytes::Bytes; +use futures::{FutureExt, future::BoxFuture}; +use http_body_util::{BodyExt, Limited}; +use reqwest::header::{AUTHORIZATION, USER_AGENT}; +use reqwest::{Body, Client, Request, RequestBuilder, Response, StatusCode}; +use secrecy::{ExposeSecret, SecretString}; +use std::time::{Duration, SystemTime}; +use tracing as log; + +use crate::GITHUB_API_VERSION; + +/// Finds the token in the user's environment, panicking if no suitable token +/// can be found. +pub fn default_token_from_env() -> SecretString { + std::env::var("GITHUB_TOKEN") + // kept for retrocompatibility but usage is discouraged and will be deprecated + .or_else(|_| std::env::var("GITHUB_API_TOKEN")) + .or_else(|_| get_token_from_git_config()) + .expect("could not find token in GITHUB_TOKEN, GITHUB_API_TOKEN or .gitconfig/github.oath-token") + .into() +} + +fn get_token_from_git_config() -> anyhow::Result { + let output = std::process::Command::new("git") + .arg("config") + .arg("--get") + .arg("github.oauth-token") + .output()?; + if !output.status.success() { + anyhow::bail!("error received executing `git`: {:?}", output.status); + } + let git_token = String::from_utf8(output.stdout)?.trim().to_string(); + Ok(git_token) +} + +#[derive(Clone)] +pub struct GithubClient { + token: SecretString, + client: Client, + pub(in crate::github) api_url: String, + /// If `true`, requests will sleep if it hits GitHub's rate limit. + retry_rate_limit: bool, +} + +#[derive(Debug, serde::Deserialize)] +pub struct RateLimitResources { + pub core: RateLimit, + pub search: RateLimit, + pub graphql: RateLimit, +} + +#[derive(Debug, serde::Deserialize)] +pub struct RateLimit { + pub limit: u64, + pub remaining: u64, + pub reset: u64, +} + +impl GithubClient { + pub fn new(token: SecretString, api_url: String) -> Self { + GithubClient { + client: Client::new(), + token, + api_url, + retry_rate_limit: false, + } + } + + pub fn new_from_env() -> Self { + Self::new( + default_token_from_env(), + std::env::var("GITHUB_API_URL") + .unwrap_or_else(|_| "https://api.github.com".to_string()), + ) + } + + /// Sets whether or not this client will retry when it hits GitHub's rate limit. + /// + /// Just beware that the retry may take a long time (like 30 minutes, + /// depending on various factors). + pub fn set_retry_rate_limit(&mut self, retry: bool) { + self.retry_rate_limit = retry; + } + + pub fn raw(&self) -> &Client { + &self.client + } + + pub async fn send_req(&self, req: RequestBuilder) -> anyhow::Result<(Bytes, String)> { + const MAX_DEFAULT_RESPONSE_SIZE: usize = 8 * 1024 * 1024; // 8 Mib + + self.send_req_with_limit(req, MAX_DEFAULT_RESPONSE_SIZE) + .await + } + + pub async fn send_req_with_limit( + &self, + req: RequestBuilder, + max_response_size: usize, + ) -> anyhow::Result<(Bytes, String)> { + const MAX_ATTEMPTS: u32 = 2; + + log::debug!("send_req with {:?}", req); + + let req_dbg = format!("{req:?}"); + + let req = req + .build() + .with_context(|| format!("building reqwest {req_dbg}"))?; + + let req_url = req.url().to_string(); + + let mut resp = self.client.execute(req.try_clone().unwrap()).await?; + if self.retry_rate_limit + && let Some(sleep) = Self::needs_retry(&resp).await + { + resp = self.retry(req, sleep, MAX_ATTEMPTS).await?; + } + + let maybe_err = resp.error_for_status_ref().err(); + let github_request_id = resp.headers().get("x-github-request-id").cloned(); + + let resp: http::Response = resp.into(); + let limited = Limited::new(resp, max_response_size); + + let body = match limited.collect().await { + Ok(body) => body.to_bytes(), + Err(e) => match e.downcast::() { + Ok(e) => { + return Err(anyhow::Error::new(*e)).with_context(|| { + format!( + "req={req_url} (x-github-request-id: {}): max response size exceeded (over {max_response_size} bytes)", + github_request_id + .as_ref() + .and_then(|v| v.to_str().ok()) + .unwrap_or("unknown") + ) + }); + } + Err(e) => { + return Err(anyhow::Error::from_boxed(e)).with_context(|| { + format!( + "req={req_url} (x-github-request-id: {}): unable to complete the request", + github_request_id + .as_ref() + .and_then(|v| v.to_str().ok()) + .unwrap_or("unknown") + ) + }); + } + }, + }; + + if let Some(e) = maybe_err { + return Err(anyhow::Error::new(e)).with_context(|| { + format!( + "req={req_url} (x-github-request-id: {}): {:.500}", + github_request_id + .as_ref() + .and_then(|v| v.to_str().ok()) + .unwrap_or("unknown"), + String::from_utf8_lossy(&body), + ) + }); + } + + Ok((body, req_dbg)) + } + + async fn needs_retry(resp: &Response) -> Option { + const REMAINING: &str = "X-RateLimit-Remaining"; + const RESET: &str = "X-RateLimit-Reset"; + + if !matches!( + resp.status(), + StatusCode::FORBIDDEN | StatusCode::TOO_MANY_REQUESTS + ) { + return None; + } + + let headers = resp.headers(); + if !(headers.contains_key(REMAINING) && headers.contains_key(RESET)) { + return None; + } + + let reset_time = headers[RESET].to_str().unwrap().parse::().unwrap(); + Some(Duration::from_secs(Self::calc_sleep(reset_time) + 10)) + } + + fn calc_sleep(reset_time: u64) -> u64 { + let epoch_time = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs(); + reset_time.saturating_sub(epoch_time) + } + + fn retry( + &self, + req: Request, + sleep: Duration, + remaining_attempts: u32, + ) -> BoxFuture<'_, Result> { + #[derive(Debug, serde::Deserialize)] + struct RateLimitResponse { + resources: RateLimitResources, + } + + log::warn!( + "Retrying after {} seconds, remaining attepts {}", + sleep.as_secs(), + remaining_attempts, + ); + + async move { + tokio::time::sleep(sleep).await; + + // check rate limit + let rate_resp = self + .client + .execute( + self.client + .get(format!("{}/rate_limit", self.api_url)) + .configure(self) + .build() + .unwrap(), + ) + .await?; + rate_resp.error_for_status_ref()?; + let rate_limit_response = rate_resp.json::().await?; + + // Check url for search path because github has different rate limits for the search api + let rate_limit = if req + .url() + .path_segments() + .is_some_and(|mut segments| segments.next() == Some("search")) + { + rate_limit_response.resources.search + } else { + rate_limit_response.resources.core + }; + + // If we still don't have any more remaining attempts, try sleeping for the remaining + // period of time + if rate_limit.remaining == 0 { + let sleep = Self::calc_sleep(rate_limit.reset); + if sleep > 0 { + tokio::time::sleep(Duration::from_secs(sleep)).await; + } + } + + let resp = self.client.execute(req.try_clone().unwrap()).await?; + if let Some(sleep) = Self::needs_retry(&resp).await + && remaining_attempts > 0 + { + return self.retry(req, sleep, remaining_attempts - 1).await; + } + + Ok(resp) + } + .boxed() + } + + pub async fn json(&self, req: RequestBuilder) -> anyhow::Result + where + T: serde::de::DeserializeOwned, + { + let (body, _req_dbg) = self.send_req(req).await?; + Ok(serde_json::from_slice(&body)?) + } + + pub fn get(&self, url: &str) -> RequestBuilder { + log::trace!("get {:?}", url); + self.client.get(url).configure(self) + } + + pub fn patch(&self, url: &str) -> RequestBuilder { + log::trace!("patch {:?}", url); + self.client.patch(url).configure(self) + } + + pub fn delete(&self, url: &str) -> RequestBuilder { + log::trace!("delete {:?}", url); + self.client.delete(url).configure(self) + } + + pub fn post(&self, url: &str) -> RequestBuilder { + log::trace!("post {:?}", url); + self.client.post(url).configure(self) + } + + pub fn put(&self, url: &str) -> RequestBuilder { + log::trace!("put {:?}", url); + self.client.put(url).configure(self) + } + + /// Fetch current rate limit, remaining and used + pub async fn rate_limit(&self) -> anyhow::Result { + #[derive(Debug, serde::Deserialize)] + struct RateLimitResponse { + resources: RateLimitResources, + } + + let url = format!("{}/rate_limit", self.api_url); + let response = self + .json::(self.get(&url)) + .await + .context("failed to fetch GitHub /rate_limit endpoint")?; + + Ok(response.resources) + } +} + +trait RequestSend: Sized { + fn configure(self, g: &GithubClient) -> Self; +} + +impl RequestSend for RequestBuilder { + fn configure(self, g: &GithubClient) -> RequestBuilder { + let mut auth = reqwest::header::HeaderValue::from_maybe_shared(format!( + "token {}", + g.token.expose_secret() + )) + .unwrap(); + auth.set_sensitive(true); + self.header(USER_AGENT, "rust-lang-triagebot") + .header(AUTHORIZATION, &auth) + .header("X-GitHub-Api-Version", GITHUB_API_VERSION) + } +} diff --git a/utils/compiler-p-high/src/github/issue.rs b/utils/compiler-p-high/src/github/issue.rs new file mode 100644 index 000000000..56824ccfa --- /dev/null +++ b/utils/compiler-p-high/src/github/issue.rs @@ -0,0 +1,267 @@ +use chrono::Utc; +use std::fmt; +use std::sync::OnceLock; +use tracing as log; + +use super::client::GithubClient; +use super::repos::{GitHubUser, Repository}; +use super::utils::opt_string; +use crate::github::GithubCommit; + +/// An issue or pull request. +/// +/// For convenience, since issues and pull requests share most of their +/// fields, this struct is used for both. The `pull_request` field can be used +/// to determine which it is. Some fields are only available on pull requests +/// (but not always, check the GitHub API for details). +#[derive(Debug, serde::Deserialize)] +pub struct Issue { + pub number: u64, + #[serde(deserialize_with = "opt_string")] + pub body: String, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, + pub title: String, + /// The common URL for viewing this issue or PR. + /// + /// Example: `https://github.com/octocat/Hello-World/pull/1347` + pub html_url: String, + // User performing an `action` (or PR/issue author) + pub user: GitHubUser, + pub labels: Vec