From dd745557a9d857ddbfc91b5c444f33f2be0f0343 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Sun, 12 Apr 2026 11:46:09 +0800 Subject: [PATCH 01/25] Refactor framework.component --- Cargo.lock | 25 + Cargo.toml | 2 + cli/Cargo.toml | 1 + cli/src/direct_runtime/runtime.rs | 2 +- cli/src/framework_component/http.rs | 733 -------- cli/src/main.rs | 39 +- cli/src/mixed_run.rs | 865 ++++++--- cli/src/mixed_run/launch_bundle.rs | 353 ++-- cli/src/mixed_run/supervisor.rs | 80 +- cli/src/mixed_run/tests.rs | 281 ++- cli/src/tests.rs | 4 +- cli/tests/linux_vm.rs | 122 +- cli/tests/mixed_run/smoke.rs | 116 +- cli/tests/mixed_run/support.rs | 176 +- compiler/src/targets/direct/mod.rs | 14 +- .../mesh/kubernetes/tests/rendering.rs | 4 +- compiler/src/targets/mesh/mesh_config.rs | 18 +- docker/amber-cli/Dockerfile | 12 +- docker/amber-docker-gateway/Dockerfile | 5 +- docker/amber-helper/Dockerfile | 5 +- docker/amber-manager/Dockerfile | 5 +- docker/amber-provisioner/Dockerfile | 5 +- docker/amber-router/Dockerfile | 5 +- runtime/mesh/src/dynamic_caps.rs | 5 +- runtime/mesh/src/lib.rs | 8 +- runtime/proxy/src/lib.rs | 9 +- runtime/router/src/control.rs | 26 +- runtime/router/src/dynamic_caps.rs | 71 +- runtime/router/src/external.rs | 2 +- runtime/router/src/lib.rs | 2 +- runtime/router/src/tests.rs | 87 +- runtime/router/src/transport.rs | 15 + runtime/site-controller/Cargo.toml | 29 + .../site-controller/src}/api.rs | 0 .../src/bin/amber-site-controller.rs | 75 + .../site-controller/src}/ccs_api.rs | 30 +- .../site-controller/src}/control_state_api.rs | 27 +- .../site-controller/src}/control_state_mcp.rs | 116 +- .../site-controller/src}/dynamic_caps.rs | 35 +- runtime/site-controller/src/http.rs | 164 ++ .../site-controller/src/lib.rs | 48 +- .../site-controller/src}/mcp.rs | 63 +- .../site-controller/src}/mcp_common.rs | 0 .../site-controller/src}/orchestration.rs | 545 ++++-- .../site-controller/src}/planner.rs | 25 +- runtime/site-controller/src/runtime_api.rs | 458 +++++ .../site-controller/src/site_controller.rs | 1202 +++++++++++++ .../site-controller/src}/state.rs | 533 ++++-- .../site-controller/src}/tests.rs | 1549 ++++++++++++----- 49 files changed, 5608 insertions(+), 2388 deletions(-) delete mode 100644 cli/src/framework_component/http.rs create mode 100644 runtime/site-controller/Cargo.toml rename {cli/src/framework_component => runtime/site-controller/src}/api.rs (100%) create mode 100644 runtime/site-controller/src/bin/amber-site-controller.rs rename {cli/src/framework_component => runtime/site-controller/src}/ccs_api.rs (73%) rename {cli/src/framework_component => runtime/site-controller/src}/control_state_api.rs (92%) rename {cli/src/framework_component => runtime/site-controller/src}/control_state_mcp.rs (72%) rename {cli/src/framework_component => runtime/site-controller/src}/dynamic_caps.rs (98%) create mode 100644 runtime/site-controller/src/http.rs rename cli/src/framework_component/mod.rs => runtime/site-controller/src/lib.rs (63%) rename {cli/src/framework_component => runtime/site-controller/src}/mcp.rs (91%) rename {cli/src/framework_component => runtime/site-controller/src}/mcp_common.rs (100%) rename {cli/src/framework_component => runtime/site-controller/src}/orchestration.rs (77%) rename {cli/src/framework_component => runtime/site-controller/src}/planner.rs (99%) create mode 100644 runtime/site-controller/src/runtime_api.rs create mode 100644 runtime/site-controller/src/site_controller.rs rename {cli/src/framework_component => runtime/site-controller/src}/state.rs (68%) rename {cli/src/framework_component => runtime/site-controller/src}/tests.rs (84%) diff --git a/Cargo.lock b/Cargo.lock index ab4b4177..aa6d20cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -110,6 +110,7 @@ dependencies = [ "amber-resolver", "amber-router", "amber-scenario", + "amber-site-controller", "amber-template", "axum", "base64", @@ -415,6 +416,30 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "amber-site-controller" +version = "0.0.0" +dependencies = [ + "amber-compiler", + "amber-manifest", + "amber-mesh", + "amber-proxy", + "amber-resolver", + "amber-scenario", + "axum", + "base64", + "miette", + "regex", + "reqwest 0.13.2", + "rmcp", + "serde", + "serde_json", + "serde_yaml", + "tempfile", + "tokio", + "url", +] + [[package]] name = "amber-template" version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml index 16a83c1c..44b23348 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "runtime/proxy", "runtime/provisioner", "runtime/router", + "runtime/site-controller", ] resolver = "2" @@ -30,6 +31,7 @@ amber-proxy = { path = "runtime/proxy" } amber-resolver = { path = "compiler/resolver" } amber-router = { path = "runtime/router" } amber-scenario = { path = "compiler/scenario" } +amber-site-controller = { path = "runtime/site-controller" } amber-template = { path = "compiler/template" } base64 = "0.22.1" futures = "0.3.31" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index aa996ccf..1e60f126 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -16,6 +16,7 @@ amber-proxy = { workspace = true } amber-resolver = { workspace = true } amber-router = { workspace = true } amber-scenario = { workspace = true } +amber-site-controller = { workspace = true } amber-template = { workspace = true } axum = { version = "0.8.6", features = ["http1", "json", "tokio"] } base64 = { workspace = true } diff --git a/cli/src/direct_runtime/runtime.rs b/cli/src/direct_runtime/runtime.rs index 5d8d4a1c..735adcae 100644 --- a/cli/src/direct_runtime/runtime.rs +++ b/cli/src/direct_runtime/runtime.rs @@ -1219,7 +1219,7 @@ pub(crate) fn rewrite_peer_addr_for_slirp_gateway(peer_addr: &str) -> String { #[cfg(target_os = "linux")] pub(crate) fn rewrite_sidecar_env_passthrough_for_slirp(name: &str, value: &str) -> String { - if name != amber_mesh::DYNAMIC_CAPS_CONTROL_URL_ENV { + if name != amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV { return value.to_string(); } rewrite_loopback_url_for_slirp_gateway(value) diff --git a/cli/src/framework_component/http.rs b/cli/src/framework_component/http.rs deleted file mode 100644 index 3559efcd..00000000 --- a/cli/src/framework_component/http.rs +++ /dev/null @@ -1,733 +0,0 @@ -use super::{ - api::{authorize_capability_instance, capability_instance_record}, - ccs_api::{ - self, FrameworkComponentInspectRequest, FrameworkComponentInspectResponse, - FrameworkComponentMutateRequest, FrameworkComponentMutateResponse, - }, - control_state_api::{ - self, DynamicCapsInspectRequest, DynamicCapsInspectResponse, DynamicCapsMutateRequest, - DynamicCapsMutateResponse, - }, - orchestration::*, - planner::*, - state::*, - *, -}; - -pub(crate) async fn run_framework_control_state(plan_path: PathBuf) -> Result<()> { - let plan: FrameworkControlStateServicePlan = - read_json(plan_path.as_path(), "framework control-state plan")?; - let mut control_state: FrameworkControlState = - read_json(Path::new(&plan.state_path), "framework control-state file")?; - persist_control_state(Path::new(&plan.state_path), &mut control_state)?; - let app_state = ControlStateApp { - control_state: Arc::new(Mutex::new(control_state)), - client: ReqwestClient::new(), - state_path: PathBuf::from(&plan.state_path), - run_root: PathBuf::from(&plan.run_root), - state_root: PathBuf::from(&plan.state_root), - mesh_scope: Arc::::from(plan.mesh_scope.clone()), - control_state_auth_token: Arc::::from(plan.auth_token), - authority_locks: Arc::new(Mutex::new(BTreeMap::new())), - bridge_proxies: Arc::new(Mutex::new(BTreeMap::new())), - }; - recover_control_state(&app_state).await?; - let app = Router::new() - .nest_service("/mcp", control_state_mcp::service(app_state.clone())) - .route("/", get(healthz)) - .route("/healthz", get(healthz)) - .route(CONTROL_SERVICE_PATH, get(get_control_state)) - .route("/v1/control-state/children", post(control_create_child)) - .route( - "/v1/control-state/children/{child}/destroy", - post(control_destroy_child), - ) - .route( - "/v1/control-state/dynamic-caps/held", - post(control_dynamic_held_list), - ) - .route( - "/v1/control-state/dynamic-caps/held/detail", - post(control_dynamic_held_detail), - ) - .route( - "/v1/control-state/dynamic-caps/share", - post(control_dynamic_share), - ) - .route( - "/v1/control-state/dynamic-caps/inspect-ref", - post(control_dynamic_inspect_ref), - ) - .route( - "/v1/control-state/dynamic-caps/revoke", - post(control_dynamic_revoke), - ) - .route( - "/v1/control-state/dynamic-caps/resolve-origin", - post(control_dynamic_resolve_origin), - ) - .with_state(app_state.clone()); - let listener = TcpListener::bind(plan.listen_addr) - .await - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to bind framework control-state service on {}", - plan.listen_addr - ) - })?; - let serve_result = axum::serve(listener, app.into_make_service()) - .with_graceful_shutdown(shutdown_signal()) - .await - .into_diagnostic(); - let cleanup_result = cleanup_dynamic_bridge_proxies(&app_state).await; - match (serve_result, cleanup_result) { - (Ok(()), Ok(())) => Ok(()), - (Err(err), Ok(())) => Err(err).wrap_err("framework control-state service failed"), - (Ok(()), Err(err)) => { - Err(err).wrap_err("framework control-state service failed to stop bridge proxies") - } - (Err(serve_err), Err(cleanup_err)) => Err(miette::miette!( - "framework control-state service failed: {serve_err}\nbridge proxy cleanup failed: \ - {cleanup_err}" - )), - } -} - -pub(crate) async fn run_framework_ccs(plan_path: PathBuf) -> Result<()> { - let plan: FrameworkCcsPlan = read_json(plan_path.as_path(), "framework CCS plan")?; - let app_state = CcsApp { - client: ReqwestClient::new(), - site_state_root: PathBuf::from(&plan.site_state_root), - control_state_url: Arc::::from(plan.control_state_url), - router_auth_token: Arc::::from(plan.router_auth_token), - control_state_auth_token: Arc::::from(plan.control_state_auth_token), - }; - let app = Router::new() - .nest_service("/mcp", mcp::service(app_state.clone())) - .route("/", get(healthz)) - .route("/healthz", get(healthz)) - .route("/v1/templates", get(ccs_list_templates)) - .route("/v1/templates/{template}", get(ccs_describe_template)) - .route( - "/v1/templates/{template}/resolve", - post(ccs_resolve_template), - ) - .route( - "/v1/children", - get(ccs_list_children).post(ccs_create_child), - ) - .route( - "/v1/children/{child}", - get(ccs_describe_child).delete(ccs_destroy_child), - ) - .route("/v1/snapshot", post(ccs_snapshot)) - .route( - "/v1/internal/dynamic-caps/origins/publish", - post(ccs_publish_dynamic_capability_origin), - ) - .with_state(app_state); - let listener = TcpListener::bind(plan.listen_addr) - .await - .into_diagnostic() - .wrap_err_with(|| format!("failed to bind framework CCS on {}", plan.listen_addr))?; - axum::serve(listener, app.into_make_service()) - .with_graceful_shutdown(shutdown_signal()) - .await - .into_diagnostic() - .wrap_err("framework CCS failed") -} - -pub(super) async fn healthz() -> Json { - Json(json!({ "ok": true })) -} - -pub(super) async fn cleanup_dynamic_bridge_proxies(app: &ControlStateApp) -> Result<()> { - let mut bridge_proxies = { - let mut guard = app.bridge_proxies.lock().await; - std::mem::take(&mut *guard) - }; - stop_bridge_proxies(&mut bridge_proxies).await -} - -pub(super) async fn get_control_state( - State(app): State, - headers: HeaderMap, -) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - Ok(Json(app.control_state.lock().await.clone())) -} - -pub(super) async fn control_create_child( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - let authority_realm_id = { - let state = app.control_state.lock().await; - capability_instance_record(&state, &request.cap_instance_id) - .map_err(ProtocolApiError::from)? - .authority_realm_id - }; - Ok(Json( - execute_create_child(&app, authority_realm_id, request.request).await?, - )) -} - -pub(super) async fn control_destroy_child( - State(app): State, - headers: HeaderMap, - AxumPath(child): AxumPath, - Json(request): Json, -) -> std::result::Result { - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - let authority_realm_id = { - let state = app.control_state.lock().await; - capability_instance_record(&state, &request.cap_instance_id) - .map_err(ProtocolApiError::from)? - .authority_realm_id - }; - execute_destroy_child(&app, authority_realm_id, &child).await?; - Ok(StatusCode::NO_CONTENT) -} - -pub(super) async fn control_dynamic_held_list( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - match control_state_api::execute_dynamic_caps_inspect( - &app, - DynamicCapsInspectRequest::HeldList(request), - ) - .await? - { - DynamicCapsInspectResponse::HeldList(response) => Ok(Json(response)), - _ => unreachable!("held_list should return held list"), - } -} - -pub(super) async fn control_dynamic_held_detail( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - match control_state_api::execute_dynamic_caps_inspect( - &app, - DynamicCapsInspectRequest::HeldDetail(request), - ) - .await? - { - DynamicCapsInspectResponse::HeldDetail(response) => Ok(Json(response)), - _ => unreachable!("held_detail should return held detail"), - } -} - -pub(super) async fn control_dynamic_share( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - match control_state_api::execute_dynamic_caps_mutate( - &app, - DynamicCapsMutateRequest::Share(request), - ) - .await? - { - DynamicCapsMutateResponse::Share(response) => Ok(Json(response)), - _ => unreachable!("share should return share response"), - } -} - -pub(super) async fn control_dynamic_inspect_ref( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - match control_state_api::execute_dynamic_caps_inspect( - &app, - DynamicCapsInspectRequest::InspectRef(request), - ) - .await? - { - DynamicCapsInspectResponse::InspectRef(response) => Ok(Json(response)), - _ => unreachable!("inspect_ref should return inspect response"), - } -} - -pub(super) async fn control_dynamic_revoke( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - match control_state_api::execute_dynamic_caps_mutate( - &app, - DynamicCapsMutateRequest::Revoke(request), - ) - .await? - { - DynamicCapsMutateResponse::Revoke(response) => Ok(Json(response)), - _ => unreachable!("revoke should return revoke response"), - } -} - -pub(super) async fn control_dynamic_resolve_origin( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> -{ - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - match control_state_api::execute_dynamic_caps_inspect( - &app, - DynamicCapsInspectRequest::ResolveOrigin(request), - ) - .await? - { - DynamicCapsInspectResponse::ResolveOrigin(response) => Ok(Json(response)), - _ => unreachable!("resolve_origin should return origin resolution"), - } -} - -pub(super) async fn ccs_list_templates( - State(app): State, - headers: HeaderMap, -) -> std::result::Result, ProtocolApiError> { - let (record, state) = authorize_request(&app, &headers).await?; - match ccs_api::execute_framework_component_inspect( - &state, - record.authority_realm_id, - FrameworkComponentInspectRequest::ListTemplates, - ) - .await? - { - FrameworkComponentInspectResponse::ListTemplates(response) => Ok(Json(response)), - _ => unreachable!("list_templates should return template list"), - } -} - -pub(super) async fn ccs_describe_template( - State(app): State, - headers: HeaderMap, - AxumPath(template): AxumPath, -) -> std::result::Result, ProtocolApiError> { - let (record, state) = authorize_request(&app, &headers).await?; - match ccs_api::execute_framework_component_inspect( - &state, - record.authority_realm_id, - FrameworkComponentInspectRequest::GetTemplate { template }, - ) - .await? - { - FrameworkComponentInspectResponse::GetTemplate(response) => Ok(Json(response)), - _ => unreachable!("get_template should return template description"), - } -} - -pub(super) async fn ccs_resolve_template( - State(app): State, - headers: HeaderMap, - AxumPath(template): AxumPath, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> { - let (record, state) = authorize_request(&app, &headers).await?; - match ccs_api::execute_framework_component_inspect( - &state, - record.authority_realm_id, - FrameworkComponentInspectRequest::ResolveTemplate { template, request }, - ) - .await? - { - FrameworkComponentInspectResponse::ResolveTemplate(response) => Ok(Json(response)), - _ => unreachable!("resolve_template should return resolved template"), - } -} - -pub(super) async fn ccs_list_children( - State(app): State, - headers: HeaderMap, -) -> std::result::Result, ProtocolApiError> { - let (record, state) = authorize_request(&app, &headers).await?; - match ccs_api::execute_framework_component_inspect( - &state, - record.authority_realm_id, - FrameworkComponentInspectRequest::ListChildren, - ) - .await? - { - FrameworkComponentInspectResponse::ListChildren(response) => Ok(Json(response)), - _ => unreachable!("list_children should return child list"), - } -} - -pub(super) async fn ccs_create_child( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> { - let (record, _) = authorize_request(&app, &headers).await?; - match ccs_api::execute_framework_component_mutate( - &app, - &record.cap_instance_id, - FrameworkComponentMutateRequest::CreateChild(request), - ) - .await? - { - FrameworkComponentMutateResponse::CreateChild(response) => Ok(Json(response)), - _ => unreachable!("create_child should return create response"), - } -} - -pub(super) async fn ccs_describe_child( - State(app): State, - headers: HeaderMap, - AxumPath(child): AxumPath, -) -> std::result::Result, ProtocolApiError> { - let (record, state) = authorize_request(&app, &headers).await?; - match ccs_api::execute_framework_component_inspect( - &state, - record.authority_realm_id, - FrameworkComponentInspectRequest::GetChild { child }, - ) - .await? - { - FrameworkComponentInspectResponse::GetChild(response) => Ok(Json(response)), - _ => unreachable!("get_child should return child description"), - } -} - -pub(super) async fn ccs_snapshot( - State(app): State, - headers: HeaderMap, -) -> std::result::Result, ProtocolApiError> { - let (record, state) = authorize_request(&app, &headers).await?; - match ccs_api::execute_framework_component_inspect( - &state, - record.authority_realm_id, - FrameworkComponentInspectRequest::GetSnapshot, - ) - .await? - { - FrameworkComponentInspectResponse::GetSnapshot(response) => Ok(Json(response)), - _ => unreachable!("get_snapshot should return snapshot"), - } -} - -pub(super) async fn ccs_publish_dynamic_capability_origin( - State(app): State, - headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> -{ - authorize_framework_auth_header(&headers, app.control_state_auth_token.as_ref())?; - Ok(Json( - publish_dynamic_capability_origin_local(&app, request).await?, - )) -} - -pub(super) async fn ccs_destroy_child( - State(app): State, - headers: HeaderMap, - AxumPath(child): AxumPath, -) -> std::result::Result { - let (record, _) = authorize_request(&app, &headers).await?; - let _ = ccs_api::execute_framework_component_mutate( - &app, - &record.cap_instance_id, - FrameworkComponentMutateRequest::DestroyChild { child }, - ) - .await?; - Ok(StatusCode::NO_CONTENT) -} - -pub(super) async fn authorize_request( - app: &CcsApp, - headers: &HeaderMap, -) -> std::result::Result<(CapabilityInstanceRecord, FrameworkControlState), ProtocolApiError> { - authorize_framework_auth_header(headers, app.router_auth_token.as_ref())?; - let route_id = required_header(headers, FRAMEWORK_ROUTE_ID_HEADER)?; - let peer_id = required_header(headers, FRAMEWORK_PEER_ID_HEADER)?; - let state = fetch_control_state(app).await?; - let record = authorize_capability_instance(&state, &route_id, &peer_id) - .map_err(ProtocolApiError::from)? - .clone(); - Ok((record, state)) -} - -pub(super) fn authorize_framework_auth_header( - headers: &HeaderMap, - expected: &str, -) -> std::result::Result<(), ProtocolApiError> { - let actual = required_header(headers, FRAMEWORK_AUTH_HEADER)?; - if actual != expected { - return Err(ProtocolApiError::unauthorized( - "invalid authenticated framework request header", - )); - } - Ok(()) -} - -pub(super) fn required_header( - headers: &HeaderMap, - name: &str, -) -> std::result::Result { - headers - .get(name) - .and_then(|value| value.to_str().ok()) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(ToOwned::to_owned) - .ok_or_else(|| { - ProtocolApiError::unauthorized(format!( - "missing authenticated framework request header `{name}`" - )) - }) -} - -pub(super) async fn fetch_control_state( - app: &CcsApp, -) -> std::result::Result { - let url = format!( - "{}{}", - app.control_state_url.trim_end_matches('/'), - CONTROL_SERVICE_PATH - ); - let response = app - .client - .get(&url) - .header(FRAMEWORK_AUTH_HEADER, app.control_state_auth_token.as_ref()) - .send() - .await - .map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "failed to reach authoritative control-state service: {err}" - )) - })?; - if !response.status().is_success() { - return Err(ProtocolApiError::control_state_unavailable(format!( - "authoritative control-state service returned {}", - response.status() - ))); - } - response.json().await.map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "authoritative control-state service returned invalid JSON: {err}" - )) - }) -} - -pub(super) async fn forward_create_child( - app: &CcsApp, - cap_instance_id: &str, - request: CreateChildRequest, -) -> std::result::Result { - let url = format!( - "{}/v1/control-state/children", - app.control_state_url.trim_end_matches('/') - ); - let response = app - .client - .post(&url) - .header(FRAMEWORK_AUTH_HEADER, app.control_state_auth_token.as_ref()) - .json(&ControlCreateChildRequest { - cap_instance_id: cap_instance_id.to_string(), - request, - }) - .send() - .await - .map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "failed to reach authoritative control-state service: {err}" - )) - })?; - parse_control_service_json(response).await -} - -pub(super) async fn forward_destroy_child( - app: &CcsApp, - cap_instance_id: &str, - child: &str, -) -> std::result::Result<(), ProtocolApiError> { - let url = format!( - "{}/v1/control-state/children/{child}/destroy", - app.control_state_url.trim_end_matches('/') - ); - let response = app - .client - .post(&url) - .header(FRAMEWORK_AUTH_HEADER, app.control_state_auth_token.as_ref()) - .json(&ControlDestroyChildRequest { - cap_instance_id: cap_instance_id.to_string(), - }) - .send() - .await - .map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "failed to reach authoritative control-state service: {err}" - )) - })?; - parse_control_service_empty(response).await -} - -pub(super) async fn parse_control_service_json Deserialize<'de>>( - response: reqwest::Response, -) -> std::result::Result { - if response.status().is_success() { - return response.json().await.map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "authoritative control-state service returned invalid JSON: {err}" - )) - }); - } - let status = response.status(); - let body = response.bytes().await.map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "failed to read authoritative control-state error response: {err}" - )) - })?; - if let Ok(protocol_error) = serde_json::from_slice::(&body) { - return Err(protocol_error.into()); - } - Err(ProtocolApiError::control_state_unavailable(format!( - "authoritative control-state service returned {status}" - ))) -} - -pub(super) async fn parse_control_service_empty( - response: reqwest::Response, -) -> std::result::Result<(), ProtocolApiError> { - if response.status().is_success() { - return Ok(()); - } - let status = response.status(); - let body = response.bytes().await.map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "failed to read authoritative control-state error response: {err}" - )) - })?; - if let Ok(protocol_error) = serde_json::from_slice::(&body) { - return Err(protocol_error.into()); - } - Err(ProtocolApiError::control_state_unavailable(format!( - "authoritative control-state service returned {status}" - ))) -} - -pub(super) async fn shutdown_signal() { - let ctrl_c = async { - signal::ctrl_c() - .await - .expect("framework service should install Ctrl-C handler"); - }; - - #[cfg(unix)] - let terminate = async { - use tokio::signal::unix::{SignalKind, signal}; - - signal(SignalKind::terminate()) - .expect("framework service should install SIGTERM handler") - .recv() - .await; - }; - - #[cfg(unix)] - tokio::select! { - _ = ctrl_c => {} - _ = terminate => {} - } - - #[cfg(not(unix))] - ctrl_c.await; -} - -pub(super) fn write_json(path: &Path, value: &T) -> Result<()> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - let bytes = serde_json::to_vec_pretty(value) - .map_err(|err| miette::miette!("failed to serialize {}: {err}", path.display()))?; - write_bytes_atomic(path, &bytes) -} - -pub(super) fn write_bytes_atomic(path: &Path, bytes: &[u8]) -> Result<()> { - let file_name = path - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or("tmp"); - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - let tmp_path = path.with_file_name(format!(".{file_name}.tmp-{}-{nonce}", std::process::id())); - - let mut file = fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&tmp_path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", tmp_path.display()))?; - if let Err(err) = file.write_all(bytes) { - let _ = fs::remove_file(&tmp_path); - return Err(miette::miette!( - "failed to write {}: {err}", - tmp_path.display() - )); - } - if let Err(err) = file.sync_all() { - let _ = fs::remove_file(&tmp_path); - return Err(miette::miette!( - "failed to sync {}: {err}", - tmp_path.display() - )); - } - drop(file); - - fs::rename(&tmp_path, path) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to replace {} with {}", - path.display(), - tmp_path.display() - ) - })?; - sync_parent_directory(path)?; - Ok(()) -} - -pub(super) fn sync_parent_directory(path: &Path) -> Result<()> { - #[cfg(unix)] - { - if let Some(parent) = path.parent() { - fs::File::open(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to open parent directory {}", parent.display()))? - .sync_all() - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to sync parent directory {}", parent.display()) - })?; - } - } - #[cfg(not(unix))] - { - let _ = path; - } - Ok(()) -} - -pub(super) fn read_json Deserialize<'de>>(path: &Path, label: &str) -> Result { - let bytes = fs::read(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {label} {}", path.display()))?; - serde_json::from_slice(&bytes) - .map_err(|err| miette::miette!("invalid {label} {}: {err}", path.display())) -} diff --git a/cli/src/main.rs b/cli/src/main.rs index fd949d7c..5df985e9 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,7 +1,6 @@ mod command_support; mod direct_runtime; mod docs; -mod framework_component; mod mixed_run; mod run_inputs; mod run_logs; @@ -423,12 +422,8 @@ enum Command { RunVmInit(RunVmInitArgs), #[command(hide = true, name = "run-site-supervisor")] RunSiteSupervisor(RunSiteSupervisorArgs), - #[command(hide = true, name = "run-site-actuator")] - RunSiteActuator(RunSiteActuatorArgs), - #[command(hide = true, name = "run-framework-control-state")] - RunFrameworkControlState(RunFrameworkControlStateArgs), - #[command(hide = true, name = "run-framework-ccs")] - RunFrameworkCcs(RunFrameworkCcsArgs), + #[command(hide = true, name = "run-site-controller")] + RunSiteController(RunSiteControllerArgs), #[command(hide = true, name = "run-detached-coordinator")] RunDetachedCoordinator(RunDetachedCoordinatorArgs), #[command(hide = true, name = "run-observability-sink")] @@ -701,22 +696,8 @@ struct RunSiteSupervisorArgs { } #[derive(Args)] -struct RunSiteActuatorArgs { - /// Path to a mixed-site site-actuator plan JSON file. - #[arg(long = "plan", value_name = "FILE")] - plan: PathBuf, -} - -#[derive(Args)] -struct RunFrameworkControlStateArgs { - /// Path to a framework control-state service plan JSON file. - #[arg(long = "plan", value_name = "FILE")] - plan: PathBuf, -} - -#[derive(Args)] -struct RunFrameworkCcsArgs { - /// Path to a framework CCS plan JSON file. +struct RunSiteControllerArgs { + /// Path to a mixed-site site-controller plan JSON file. #[arg(long = "plan", value_name = "FILE")] plan: PathBuf, } @@ -895,12 +876,12 @@ async fn main() -> Result<()> { .await } Command::RunSiteSupervisor(args) => mixed_run::run_site_supervisor(args.plan).await, - Command::RunSiteActuator(args) => mixed_run::run_site_actuator(args.plan).await, - Command::RunFrameworkControlState(args) => { - framework_component::run_framework_control_state(args.plan).await - } - Command::RunFrameworkCcs(args) => { - framework_component::run_framework_ccs(args.plan).await + Command::RunSiteController(args) => { + amber_site_controller::run_site_controller( + args.plan, + mixed_run::site_controller_runtime(), + ) + .await } Command::RunDetachedCoordinator(args) => run_detached_coordinator(args).await, Command::RunObservabilitySink(args) => { diff --git a/cli/src/mixed_run.rs b/cli/src/mixed_run.rs index a1d7bdbc..465580c5 100644 --- a/cli/src/mixed_run.rs +++ b/cli/src/mixed_run.rs @@ -37,11 +37,16 @@ use amber_proxy::{ fetch_router_identity, load_output_proxy_metadata, register_export_peer_with_retry, register_external_slot_with_retry, revoke_route_overlay_with_retry, }; -use axum::{ - Router, - extract::{Path as AxumPath, State}, - http::StatusCode, - routing::{get, post}, +use amber_site_controller::{ + DesiredExportPeerOverlay as ControllerDesiredExportPeerOverlay, + DesiredExternalSlotOverlay as ControllerDesiredExternalSlotOverlay, DynamicInputRouteRecord, + DynamicInputRouteTarget, DynamicProxyExportRecord, DynamicSitePlanRecord, + LaunchedSite as ControllerLaunchedSite, + LiveComponentRuntimeMetadata as ControllerLiveComponentRuntimeMetadata, + SharedSiteControllerRuntime, SiteControllerPlan, + SiteControllerRuntime as FrameworkSiteControllerRuntime, SiteControllerRuntimeFuture, + SiteControllerRuntimePlan as ControllerSiteControllerRuntimePlan, + SiteReceipt as ControllerSiteReceipt, }; use base64::Engine as _; use miette::{Context as _, IntoDiagnostic as _, Result}; @@ -56,7 +61,6 @@ use serde_json::{Map as JsonMap, Value as JsonValue}; use sha2::{Digest as _, Sha256}; use tokio::{ io::{AsyncReadExt as _, AsyncWriteExt as _}, - net::TcpListener as TokioTcpListener, sync::Mutex as AsyncMutex, time::{Instant, sleep}, }; @@ -67,11 +71,6 @@ use crate::{ direct_current_control_socket_path, direct_runtime::ensure_direct_control_socket_link, direct_runtime_state_path, - framework_component::{ - DynamicInputRouteRecord, DynamicInputRouteTarget, DynamicProxyExportRecord, - DynamicSitePlanRecord, SiteActuatorDestroyRequest, SiteActuatorPrepareRequest, - SiteActuatorPublishRequest, - }, run_inputs::{collect_run_interface, validate_export_bindings, validate_slot_bindings}, site_proxy_metadata::load_site_proxy_metadata, tcp_readiness::{ @@ -112,8 +111,10 @@ const SITE_STATE_SCHEMA: &str = "amber.run.site_state"; const SITE_STATE_VERSION: u32 = 2; const SITE_PLAN_SCHEMA: &str = "amber.run.site_supervisor_plan"; const SITE_PLAN_VERSION: u32 = 1; -const SITE_ACTUATOR_PLAN_SCHEMA: &str = "amber.run.site_actuator_plan"; -const SITE_ACTUATOR_PLAN_VERSION: u32 = 1; +const SITE_CONTROLLER_RUNTIME_PLAN_SCHEMA: &str = "amber.run.site_controller_runtime_plan"; +const SITE_CONTROLLER_RUNTIME_PLAN_VERSION: u32 = 1; +const SITE_CONTROLLER_RUNTIME_STATE_SCHEMA: &str = "amber.run.site_controller_runtime_state"; +const SITE_CONTROLLER_RUNTIME_STATE_VERSION: u32 = 1; const DESIRED_LINKS_SCHEMA: &str = "amber.run.desired_links"; const DESIRED_LINKS_VERSION: u32 = 1; const OTLP_SINK_PLAN_SCHEMA: &str = "amber.run.observability_sink"; @@ -147,6 +148,303 @@ const CONTAINER_HOST_ALIAS: &str = "host.docker.internal"; static MANAGER_OBSERVABILITY_ENDPOINT: OnceLock>> = OnceLock::new(); static KUBERNETES_CONTAINER_HOST_IP: OnceLock> = OnceLock::new(); +pub(crate) fn amber_cli_executable() -> Result { + let current = env::current_exe() + .into_diagnostic() + .wrap_err("failed to resolve amber executable path")?; + workspace_executable_from(¤t, "CARGO_BIN_EXE_amber", "amber", true) +} + +#[derive(Clone, Debug)] +pub(crate) struct SiteControllerCommand { + pub(crate) executable: PathBuf, + pub(crate) prefix_args: Vec<&'static str>, +} + +pub(crate) fn site_controller_command() -> Result { + let current = env::current_exe() + .into_diagnostic() + .wrap_err("failed to resolve site controller executable path")?; + site_controller_command_from(¤t) +} + +fn site_controller_command_from(current: &Path) -> Result { + match workspace_executable_from( + current, + "CARGO_BIN_EXE_amber-site-controller", + "amber-site-controller", + false, + ) { + Ok(executable) => Ok(SiteControllerCommand { + executable, + prefix_args: Vec::new(), + }), + Err(_) => Ok(SiteControllerCommand { + executable: amber_cli_executable()?, + prefix_args: vec!["run-site-controller"], + }), + } +} + +fn workspace_executable_from( + current: &Path, + env_var: &str, + binary_name: &str, + allow_current: bool, +) -> Result { + if let Some(path) = env::var_os(env_var) { + let path = PathBuf::from(path); + if path.is_file() { + return Ok(path); + } + } + + let executable_name = format!("{binary_name}{}", std::env::consts::EXE_SUFFIX); + if allow_current + && current.file_name().and_then(|name| name.to_str()) == Some(executable_name.as_str()) + { + return Ok(current.to_path_buf()); + } + if let Some(bin_dir) = current.parent() { + for dir in [Some(bin_dir), bin_dir.parent()].into_iter().flatten() { + let candidate = dir.join(&executable_name); + if candidate.is_file() { + return Ok(candidate); + } + } + } + Err(miette::miette!( + "failed to locate the `{binary_name}` executable; expected {env_var} or a sibling binary \ + next to {}", + current.display() + )) +} + +#[derive(Clone, Default)] +pub(crate) struct CliSiteControllerRuntime { + bridge_proxies: Arc>>, +} + +pub(crate) fn site_controller_runtime() -> SharedSiteControllerRuntime { + Arc::new(CliSiteControllerRuntime::default()) +} + +fn local_site_receipt(receipt: &ControllerSiteReceipt) -> SiteReceipt { + SiteReceipt { + kind: receipt.kind, + artifact_dir: receipt.artifact_dir.clone(), + supervisor_pid: receipt.supervisor_pid, + process_pid: receipt.process_pid, + compose_project: receipt.compose_project.clone(), + kubernetes_namespace: receipt.kubernetes_namespace.clone(), + port_forward_pid: receipt.port_forward_pid, + context: receipt.context.clone(), + router_control: receipt.router_control.clone(), + router_mesh_addr: receipt.router_mesh_addr.clone(), + router_identity_id: receipt.router_identity_id.clone(), + router_public_key_b64: receipt.router_public_key_b64.clone(), + site_controller_pid: receipt.site_controller_pid, + site_controller_url: receipt.site_controller_url.clone(), + } +} + +fn local_site_controller_runtime_plan( + plan: &ControllerSiteControllerRuntimePlan, +) -> SiteControllerRuntimePlan { + SiteControllerRuntimePlan { + schema: plan.schema.clone(), + version: plan.version, + run_id: plan.run_id.clone(), + mesh_scope: plan.mesh_scope.clone(), + run_root: plan.run_root.clone(), + site_id: plan.site_id.clone(), + kind: plan.kind, + router_identity_id: plan.router_identity_id.clone(), + artifact_dir: plan.artifact_dir.clone(), + site_state_root: plan.site_state_root.clone(), + listen_addr: plan.listen_addr, + storage_root: plan.storage_root.clone(), + runtime_root: plan.runtime_root.clone(), + router_mesh_port: plan.router_mesh_port, + compose_project: plan.compose_project.clone(), + kubernetes_namespace: plan.kubernetes_namespace.clone(), + context: plan.context.clone(), + observability_endpoint: plan.observability_endpoint.clone(), + launch_env: plan.launch_env.clone(), + } +} + +fn local_launched_site(site: &ControllerLaunchedSite) -> LaunchedSite { + LaunchedSite { + receipt: local_site_receipt(&site.receipt), + router_control: site.router_control.clone(), + router_identity: site.router_identity.clone(), + router_addr: site.router_addr, + } +} + +impl FrameworkSiteControllerRuntime for CliSiteControllerRuntime { + fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async move { + let mut bridge_proxies = { + let mut guard = self.bridge_proxies.lock().await; + std::mem::take(&mut *guard) + }; + stop_bridge_proxies(&mut bridge_proxies).await + }) + } + + fn resolve_link_external_url<'a>( + &'a self, + provider: &'a ControllerLaunchedSite, + provider_output_dir: &'a Path, + link: &'a RunLink, + consumer_kind: SiteKind, + run_root: &'a Path, + ) -> SiteControllerRuntimeFuture<'a, String> { + Box::pin(async move { + let provider = local_launched_site(provider); + let mut bridge_proxies = self.bridge_proxies.lock().await; + resolve_link_external_url_for_output( + &provider, + provider_output_dir, + link, + consumer_kind, + run_root, + &mut bridge_proxies, + ) + .await + }) + } + + fn prepare_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + child_id: u64, + site_plan: DynamicSitePlanRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async move { site_controller_prepare_child(plan, child_id, site_plan).await }) + } + + fn publish_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + child_id: u64, + site_plan: DynamicSitePlanRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async move { site_controller_publish_child(plan, child_id, site_plan).await }) + } + + fn rollback_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + child_id: u64, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async move { site_controller_rollback_child(plan, child_id).await }) + } + + fn destroy_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + child_id: u64, + desired_site_plan: Option, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin( + async move { site_controller_destroy_child(plan, child_id, desired_site_plan).await }, + ) + } + + fn collect_live_component_runtime_metadata( + &self, + plan: &ControllerSiteControllerRuntimePlan, + ) -> Result> { + collect_live_component_runtime_metadata(&local_site_controller_runtime_plan(plan)).map( + |components| { + components + .into_iter() + .map(|(component_id, metadata)| { + ( + component_id, + ControllerLiveComponentRuntimeMetadata { + moniker: metadata.moniker, + host_mesh_addr: metadata.host_mesh_addr, + mesh_config: metadata.mesh_config, + }, + ) + }) + .collect() + }, + ) + } + + fn load_live_site_router_mesh_config( + &self, + plan: &ControllerSiteControllerRuntimePlan, + ) -> Result { + load_live_site_router_mesh_config(&local_site_controller_runtime_plan(plan)) + } + + fn router_mesh_addr_for_consumer( + &self, + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> Result { + router_mesh_addr_for_consumer(provider_kind, consumer_kind, router_mesh_addr) + } + + fn update_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: ControllerDesiredExternalSlotOverlay, + ) -> Result<()> { + update_desired_overlay_for_consumer( + site_state_root, + overlay_id, + DesiredExternalSlotOverlay { + slot_name: overlay.slot_name, + url: overlay.url, + }, + ) + } + + fn update_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: ControllerDesiredExportPeerOverlay, + ) -> Result<()> { + update_desired_overlay_for_provider( + site_state_root, + overlay_id, + DesiredExportPeerOverlay { + export_name: overlay.export_name, + peer_id: overlay.peer_id, + peer_key_b64: overlay.peer_key_b64, + protocol: overlay.protocol, + route_id: overlay.route_id, + }, + ) + } + + fn clear_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> Result<()> { + clear_desired_overlay_for_consumer(site_state_root, overlay_id) + } + + fn clear_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> Result<()> { + clear_desired_overlay_for_provider(site_state_root, overlay_id) + } +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct RunReceipt { pub(crate) schema: String, @@ -158,20 +456,12 @@ pub(crate) struct RunReceipt { pub(crate) source_plan_path: Option, pub(crate) run_root: String, #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) framework_control_state: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) observability: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub(crate) bridge_proxies: Vec, pub(crate) sites: BTreeMap, } -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct FrameworkControlStateReceipt { - pub(crate) pid: u32, - pub(crate) url: String, -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct BridgeProxyReceipt { pub(crate) export_name: String, @@ -326,6 +616,10 @@ pub(crate) struct SiteReceipt { pub(crate) router_identity_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) router_public_key_b64: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) site_controller_pid: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) site_controller_url: Option, } #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] @@ -367,6 +661,10 @@ struct SiteManagerState { #[serde(default, skip_serializing_if = "Option::is_none")] router_public_key_b64: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + site_controller_pid: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + site_controller_url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] last_error: Option, } @@ -401,15 +699,15 @@ pub(crate) struct SiteSupervisorPlan { #[serde(default, skip_serializing_if = "Option::is_none")] observability_endpoint: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - framework_ccs_plan_path: Option, + site_controller_plan_path: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - site_actuator_plan_path: Option, + site_controller_url: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] launch_env: BTreeMap, } #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct SiteActuatorPlan { +pub(crate) struct SiteControllerRuntimePlan { pub(crate) schema: String, pub(crate) version: u32, pub(crate) run_id: String, @@ -440,14 +738,14 @@ pub(crate) struct SiteActuatorPlan { } #[derive(Clone, Debug, Serialize, Deserialize)] -struct SiteActuatorState { +struct SiteControllerRuntimeState { schema: String, version: u32, run_id: String, site_id: String, kind: SiteKind, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - children: BTreeMap, + children: BTreeMap, } #[derive(Clone, Debug)] @@ -458,7 +756,7 @@ pub(crate) struct LiveComponentRuntimeMetadata { } #[derive(Clone, Debug, Serialize, Deserialize)] -struct SiteActuatorChildRecord { +struct SiteControllerRuntimeChildRecord { child_id: u64, artifact_root: String, #[serde(default, skip_serializing_if = "Vec::is_empty")] @@ -503,8 +801,8 @@ struct SupervisorPlanInput<'a> { artifact_dir: &'a Path, site_state_root: &'a Path, observability_endpoint: Option<&'a str>, - framework_ccs_plan_path: Option<&'a Path>, - site_actuator_plan_path: Option<&'a Path>, + site_controller_plan_path: Option<&'a Path>, + site_controller_url: Option<&'a str>, } #[derive(Clone, Debug)] @@ -513,15 +811,6 @@ struct MaterializedObservability { plan_path: Option, } -#[derive(Clone, Debug)] -struct MaterializedFrameworkControlState { - plan_path: PathBuf, - receipt: FrameworkControlStateReceipt, - router_auth_token: String, - control_state_auth_token: String, - dynamic_caps_token_verify_key_b64: String, -} - #[derive(Clone, Debug)] pub(crate) struct MaterializedSite { pub(crate) site_plan: RunSitePlan, @@ -533,7 +822,6 @@ pub(crate) struct MaterializedSite { #[derive(Clone, Debug)] struct MaterializedLaunchBundle { run_plan_path: PathBuf, - framework_control_state: Option, observability: Option, sites: BTreeMap, } @@ -707,8 +995,7 @@ struct SupervisorRuntime { site_process: Option, site_started: bool, port_forward: Option, - framework_ccs: Option, - site_actuator: Option, + site_controller: Option, last_start_attempt: Option, last_stitch_refresh: Option, ready_since: Option, @@ -749,6 +1036,8 @@ fn site_state_from_receipt( router_mesh_addr: site.router_mesh_addr.clone(), router_identity_id: site.router_identity_id.clone(), router_public_key_b64: site.router_public_key_b64.clone(), + site_controller_pid: site.site_controller_pid, + site_controller_url: site.site_controller_url.clone(), last_error, } } @@ -865,8 +1154,7 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { site_process: None, site_started: false, port_forward: None, - framework_ccs: None, - site_actuator: None, + site_controller: None, last_start_attempt: None, last_stitch_refresh: None, ready_since: None, @@ -908,6 +1196,8 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { router_mesh_addr: None, router_identity_id: None, router_public_key_b64: None, + site_controller_pid: None, + site_controller_url: None, last_error: None, }, )?; @@ -939,6 +1229,8 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { router_mesh_addr: None, router_identity_id: None, router_public_key_b64: None, + site_controller_pid: None, + site_controller_url: None, last_error: Some("coordinator exited before commit".to_string()), }, )?; @@ -1045,6 +1337,8 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { router_mesh_addr: discovery.router_addr.map(|addr| addr.to_string()), router_identity_id: Some(discovery.router_identity.id), router_public_key_b64: Some(public_key_b64), + site_controller_pid: None, + site_controller_url: None, last_error: None, }, )?; @@ -1130,15 +1424,16 @@ pub(crate) async fn run_observability_sink(plan_path: PathBuf) -> Result<()> { } } -pub(crate) async fn run_site_actuator(plan_path: PathBuf) -> Result<()> { - let plan: SiteActuatorPlan = read_json(&plan_path, "site actuator plan")?; - let state_path = site_actuator_state_path(Path::new(&plan.site_state_root)); +fn build_site_controller_runtime_app( + plan: SiteControllerRuntimePlan, +) -> Result { + let state_path = site_controller_runtime_state_path(Path::new(&plan.site_state_root)); let initial_state = if state_path.is_file() { - read_json(&state_path, "site actuator state")? + read_json(&state_path, "site controller runtime state")? } else { - let state = SiteActuatorState { - schema: "amber.run.site_actuator_state".to_string(), - version: 1, + let state = SiteControllerRuntimeState { + schema: SITE_CONTROLLER_RUNTIME_STATE_SCHEMA.to_string(), + version: SITE_CONTROLLER_RUNTIME_STATE_VERSION, run_id: plan.run_id.clone(), site_id: plan.site_id.clone(), kind: plan.kind, @@ -1147,142 +1442,139 @@ pub(crate) async fn run_site_actuator(plan_path: PathBuf) -> Result<()> { write_json(&state_path, &state)?; state }; - let app = SiteActuatorApp { + Ok(SiteControllerRuntimeApp { plan, state_path, state: Arc::new(AsyncMutex::new(initial_state)), - }; - let stop_requested = install_signal_flag(); - let listener = TokioTcpListener::bind(app.plan.listen_addr) - .await - .into_diagnostic() - .wrap_err_with(|| format!("failed to bind site actuator on {}", app.plan.listen_addr))?; - let router = Router::new() - .route("/healthz", get(site_actuator_healthz)) - .route( - "/v1/children/{child_id}/prepare", - post(site_actuator_prepare), - ) - .route( - "/v1/children/{child_id}/publish", - post(site_actuator_publish), - ) - .route( - "/v1/children/{child_id}/rollback", - post(site_actuator_rollback), - ) - .route( - "/v1/children/{child_id}/destroy", - post(site_actuator_destroy), - ) - .with_state(app); - axum::serve(listener, router.into_make_service()) - .with_graceful_shutdown(async move { - while !stop_requested.load(Ordering::SeqCst) { - sleep(Duration::from_millis(100)).await; - } - }) - .await - .into_diagnostic() - .wrap_err("site actuator failed") -} - -#[derive(Clone)] -struct SiteActuatorApp { - plan: SiteActuatorPlan, - state_path: PathBuf, - state: Arc>, + }) } -type ActuatorHttpResult = std::result::Result; - -async fn site_actuator_healthz() -> StatusCode { - StatusCode::NO_CONTENT +pub(crate) fn site_controller_runtime_plan_from_controller_plan( + plan: &SiteControllerPlan, +) -> SiteControllerRuntimePlan { + SiteControllerRuntimePlan { + schema: SITE_CONTROLLER_RUNTIME_PLAN_SCHEMA.to_string(), + version: SITE_CONTROLLER_RUNTIME_PLAN_VERSION, + run_id: plan.run_id.clone(), + mesh_scope: plan.mesh_scope.clone(), + run_root: plan.run_root.clone(), + site_id: plan.site_id.clone(), + kind: plan.kind, + router_identity_id: plan.router_identity_id.clone(), + artifact_dir: plan.artifact_dir.clone(), + site_state_root: plan.site_state_root.clone(), + listen_addr: plan.listen_addr, + storage_root: plan.storage_root.clone(), + runtime_root: plan.runtime_root.clone(), + router_mesh_port: plan.router_mesh_port, + compose_project: plan.compose_project.clone(), + kubernetes_namespace: plan.kubernetes_namespace.clone(), + context: plan.context.clone(), + observability_endpoint: plan.observability_endpoint.clone(), + launch_env: plan.launch_env.clone(), + } } -async fn site_actuator_prepare( - State(app): State, - AxumPath(child_id): AxumPath, - axum::Json(request): axum::Json, -) -> ActuatorHttpResult { - actuator_prepare_child(&app, child_id, request.site_plan) - .await - .map_err(actuator_error)?; - Ok(StatusCode::NO_CONTENT) +pub(crate) async fn site_controller_prepare_child( + plan: &SiteControllerPlan, + child_id: u64, + site_plan: DynamicSitePlanRecord, +) -> Result<()> { + let app = + build_site_controller_runtime_app(site_controller_runtime_plan_from_controller_plan(plan))?; + site_controller_runtime_prepare_child(&app, child_id, site_plan).await } -async fn site_actuator_publish( - State(app): State, - AxumPath(child_id): AxumPath, - axum::Json(request): axum::Json, -) -> ActuatorHttpResult { - actuator_publish_child(&app, child_id, request.site_plan) - .await - .map_err(actuator_error)?; - Ok(StatusCode::NO_CONTENT) +pub(crate) async fn site_controller_publish_child( + plan: &SiteControllerPlan, + child_id: u64, + site_plan: DynamicSitePlanRecord, +) -> Result<()> { + let app = + build_site_controller_runtime_app(site_controller_runtime_plan_from_controller_plan(plan))?; + site_controller_runtime_publish_child(&app, child_id, site_plan).await } -async fn site_actuator_rollback( - State(app): State, - AxumPath(child_id): AxumPath, -) -> ActuatorHttpResult { - actuator_rollback_child(&app, child_id) - .await - .map_err(actuator_error)?; - Ok(StatusCode::NO_CONTENT) +pub(crate) async fn site_controller_rollback_child( + plan: &SiteControllerPlan, + child_id: u64, +) -> Result<()> { + let app = + build_site_controller_runtime_app(site_controller_runtime_plan_from_controller_plan(plan))?; + site_controller_runtime_rollback_child(&app, child_id).await } -async fn site_actuator_destroy( - State(app): State, - AxumPath(child_id): AxumPath, - axum::Json(request): axum::Json, -) -> ActuatorHttpResult { - actuator_destroy_child(&app, child_id, request.desired_site_plan) - .await - .map_err(actuator_error)?; - Ok(StatusCode::NO_CONTENT) +pub(crate) async fn site_controller_destroy_child( + plan: &SiteControllerPlan, + child_id: u64, + desired_site_plan: Option, +) -> Result<()> { + let app = + build_site_controller_runtime_app(site_controller_runtime_plan_from_controller_plan(plan))?; + site_controller_runtime_destroy_child(&app, child_id, desired_site_plan).await } -fn actuator_error(err: miette::Report) -> (StatusCode, String) { - (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()) +#[derive(Clone)] +struct SiteControllerRuntimeApp { + plan: SiteControllerRuntimePlan, + state_path: PathBuf, + state: Arc>, } -fn site_actuator_state_path(site_state_root: &Path) -> PathBuf { - site_state_root.join("site-actuator-state.json") +fn site_controller_runtime_state_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("site-controller-runtime-state.json") } -fn site_actuator_child_root(plan: &SiteActuatorPlan, child_id: u64) -> PathBuf { +fn site_controller_runtime_child_root(plan: &SiteControllerRuntimePlan, child_id: u64) -> PathBuf { Path::new(&plan.site_state_root) .join("framework-component") .join("children") .join(child_id.to_string()) } -pub(super) fn site_actuator_child_root_for_site(site_state_root: &Path, child_id: u64) -> PathBuf { +pub(super) fn site_controller_runtime_child_root_for_site( + site_state_root: &Path, + child_id: u64, +) -> PathBuf { site_state_root .join("framework-component") .join("children") .join(child_id.to_string()) } -fn site_actuator_child_artifact_root(plan: &SiteActuatorPlan, child_id: u64) -> PathBuf { - site_actuator_child_root(plan, child_id).join("artifact") +fn site_controller_runtime_child_artifact_root( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> PathBuf { + site_controller_runtime_child_root(plan, child_id).join("artifact") } -fn site_actuator_child_runtime_root(plan: &SiteActuatorPlan, child_id: u64) -> PathBuf { - site_actuator_child_root(plan, child_id).join("runtime") +fn site_controller_runtime_child_runtime_root( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> PathBuf { + site_controller_runtime_child_root(plan, child_id).join("runtime") } -fn site_actuator_child_storage_root(plan: &SiteActuatorPlan, child_id: u64) -> PathBuf { - site_actuator_child_root(plan, child_id).join("storage") +fn site_controller_runtime_child_storage_root( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> PathBuf { + site_controller_runtime_child_root(plan, child_id).join("storage") } -fn site_actuator_child_peer_ports_path(plan: &SiteActuatorPlan, child_id: u64) -> PathBuf { - site_actuator_child_root(plan, child_id).join("existing-peer-ports.json") +fn site_controller_runtime_child_peer_ports_path( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> PathBuf { + site_controller_runtime_child_root(plan, child_id).join("existing-peer-ports.json") } -fn site_actuator_child_peer_identities_path(plan: &SiteActuatorPlan, child_id: u64) -> PathBuf { - site_actuator_child_root(plan, child_id).join("existing-peer-identities.json") +fn site_controller_runtime_child_peer_identities_path( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> PathBuf { + site_controller_runtime_child_root(plan, child_id).join("existing-peer-identities.json") } fn dynamic_compose_child_metadata_path(artifact_root: &Path) -> PathBuf { @@ -1341,11 +1633,12 @@ fn load_dynamic_proxy_exports_metadata( } pub(super) fn cleanup_dynamic_site_children(site_state_root: &Path, kind: SiteKind) -> Result<()> { - let state_path = site_actuator_state_path(site_state_root); + let state_path = site_controller_runtime_state_path(site_state_root); if !state_path.is_file() { return Ok(()); } - let mut state: SiteActuatorState = read_json(&state_path, "site actuator state")?; + let mut state: SiteControllerRuntimeState = + read_json(&state_path, "site controller runtime state")?; let compose_supervisor_plan = matches!(kind, SiteKind::Compose) .then(|| { read_json::( @@ -1361,7 +1654,7 @@ pub(super) fn cleanup_dynamic_site_children(site_state_root: &Path, kind: SiteKi if let Some(pid) = child.process_pid { terminate_pid(pid, site_ready_timeout_for_kind(kind))?; } - remove_dir_if_exists(&site_actuator_child_root_for_site( + remove_dir_if_exists(&site_controller_runtime_child_root_for_site( site_state_root, child.child_id, ))?; @@ -1375,7 +1668,7 @@ pub(super) fn cleanup_dynamic_site_children(site_state_root: &Path, kind: SiteKi fn cleanup_dynamic_compose_child( plan: &SiteSupervisorPlan, - child: &SiteActuatorChildRecord, + child: &SiteControllerRuntimeChildRecord, ) -> Result<()> { let artifact_root = Path::new(&child.artifact_root); if !dynamic_compose_child_metadata_path(artifact_root).is_file() { @@ -1864,8 +2157,8 @@ fn compose_dynamic_root_service_names( } fn compose_live_service_names( - plan: &SiteActuatorPlan, - published_children: &[SiteActuatorChildRecord], + plan: &SiteControllerRuntimePlan, + published_children: &[SiteControllerRuntimeChildRecord], ) -> Result> { let site_compose_path = Path::new(&plan.artifact_dir).join("compose.yaml"); let site_document = read_compose_document(&site_compose_path)?; @@ -1983,7 +2276,7 @@ fn overlay_peer_addr_map_from_ports(ports: &BTreeMap) -> BTreeMap Result>> { - dynamic_route_issuer_grants(&[SiteActuatorChildRecord { + dynamic_route_issuer_grants(&[SiteControllerRuntimeChildRecord { child_id: 0, artifact_root: String::new(), assigned_components: Vec::new(), @@ -2778,7 +3071,9 @@ fn load_dynamic_compose_child_metadata( ) } -fn load_running_site_router_identity(plan: &SiteActuatorPlan) -> Result { +fn load_running_site_router_identity( + plan: &SiteControllerRuntimePlan, +) -> Result { let state: SiteManagerState = read_json( &Path::new(&plan.site_state_root).join("manager-state.json"), "site manager state", @@ -2903,8 +3198,8 @@ fn compose_peer_identities_for_artifact( } fn local_compose_peer_identities( - plan: &SiteActuatorPlan, - published_children: &[SiteActuatorChildRecord], + plan: &SiteControllerRuntimePlan, + published_children: &[SiteControllerRuntimeChildRecord], ) -> Result> { let mut peers = BTreeMap::new(); let router = load_running_site_router_identity(plan)?; @@ -2923,7 +3218,7 @@ fn local_compose_peer_identities( } fn kubernetes_peer_identities_for_artifact( - plan: &SiteActuatorPlan, + plan: &SiteControllerRuntimePlan, artifact_root: &Path, ) -> Result> { let mesh_plan = read_embedded_kubernetes_mesh_provision_plan(artifact_root)?; @@ -2946,8 +3241,8 @@ fn kubernetes_peer_identities_for_artifact( } fn local_kubernetes_peer_identities( - plan: &SiteActuatorPlan, - published_children: &[SiteActuatorChildRecord], + plan: &SiteControllerRuntimePlan, + published_children: &[SiteControllerRuntimeChildRecord], ) -> Result> { let mut peers = BTreeMap::new(); let router = load_running_site_router_identity(plan)?; @@ -2966,11 +3261,11 @@ fn local_kubernetes_peer_identities( } pub(crate) fn collect_live_component_runtime_metadata( - plan: &SiteActuatorPlan, + plan: &SiteControllerRuntimePlan, ) -> Result> { - let state: SiteActuatorState = read_json( - &site_actuator_state_path(Path::new(&plan.site_state_root)), - "site actuator state", + let state: SiteControllerRuntimeState = read_json( + &site_controller_runtime_state_path(Path::new(&plan.site_state_root)), + "site controller runtime state", )?; let published_children = state .children @@ -3003,11 +3298,11 @@ pub(crate) fn collect_live_component_runtime_metadata( let child_components = match plan.kind { SiteKind::Direct => collect_direct_artifact_runtime_metadata( Path::new(&child.artifact_root), - &site_actuator_child_runtime_root(plan, child.child_id), + &site_controller_runtime_child_runtime_root(plan, child.child_id), )?, SiteKind::Vm => collect_vm_artifact_runtime_metadata( Path::new(&child.artifact_root), - &site_actuator_child_runtime_root(plan, child.child_id), + &site_controller_runtime_child_runtime_root(plan, child.child_id), )?, SiteKind::Compose => collect_compose_artifact_runtime_metadata( Path::new(&child.artifact_root), @@ -3023,7 +3318,7 @@ pub(crate) fn collect_live_component_runtime_metadata( } pub(crate) fn load_live_site_router_mesh_config( - plan: &SiteActuatorPlan, + plan: &SiteControllerRuntimePlan, ) -> Result { let artifact_root = Path::new(&plan.artifact_dir); match plan.kind { @@ -3262,7 +3557,7 @@ fn collect_compose_artifact_runtime_metadata( } fn collect_kubernetes_artifact_runtime_metadata( - plan: &SiteActuatorPlan, + plan: &SiteControllerRuntimePlan, artifact_root: &Path, ) -> Result> { let mesh_plan = read_kubernetes_runtime_mesh_provision_plan(artifact_root)?; @@ -3305,10 +3600,10 @@ fn collect_kubernetes_artifact_runtime_metadata( } fn prepare_dynamic_compose_child_artifact( - plan: &SiteActuatorPlan, + plan: &SiteControllerRuntimePlan, site_plan: &DynamicSitePlanRecord, artifact_root: &Path, - published_children: &[SiteActuatorChildRecord], + published_children: &[SiteControllerRuntimeChildRecord], existing_site_peer_identities: &BTreeMap, ) -> Result<()> { project_dynamic_child_mesh_scope(artifact_root, Some(&plan.mesh_scope))?; @@ -3813,7 +4108,7 @@ fn rewrite_dynamic_kubernetes_apply_bundle( } fn prepare_dynamic_kubernetes_child_artifact( - plan: &SiteActuatorPlan, + plan: &SiteControllerRuntimePlan, site_plan: &DynamicSitePlanRecord, artifact_root: &Path, existing_site_peer_identities: &BTreeMap, @@ -4521,7 +4816,7 @@ fn reconcile_artifact_files(site_plan: &DynamicSitePlanRecord) -> &BTreeMap, ) -> Result<(String, KubernetesSecretPayload)> { @@ -4606,7 +4901,7 @@ where } fn load_kubernetes_mesh_identity_secret( - plan: &SiteActuatorPlan, + plan: &SiteControllerRuntimePlan, name: &str, namespace: Option<&str>, ) -> Result { @@ -4621,7 +4916,7 @@ fn load_kubernetes_mesh_identity_secret( } fn load_kubernetes_mesh_config_public( - plan: &SiteActuatorPlan, + plan: &SiteControllerRuntimePlan, name: &str, namespace: Option<&str>, ) -> Result { @@ -4654,8 +4949,8 @@ fn reconcile_site_proxy_metadata( } fn project_dynamic_direct_router_surface( - plan: &SiteActuatorPlan, - child: &SiteActuatorChildRecord, + plan: &SiteControllerRuntimePlan, + child: &SiteControllerRuntimeChildRecord, ) -> Result<()> { let state_path = direct_runtime_state_path(Path::new(&child.artifact_root)); let router_mesh_port = plan.router_mesh_port.ok_or_else(|| { @@ -4716,13 +5011,13 @@ fn project_dynamic_direct_router_surface( } fn project_dynamic_vm_router_surface( - plan: &SiteActuatorPlan, - child: &SiteActuatorChildRecord, + plan: &SiteControllerRuntimePlan, + child: &SiteControllerRuntimeChildRecord, ) -> Result<()> { let state_path = Path::new(&child.artifact_root) .join(".amber") .join("vm-runtime.json"); - let runtime_root = site_actuator_child_runtime_root(plan, child.child_id); + let runtime_root = site_controller_runtime_child_runtime_root(plan, child.child_id); let router_mesh_port = plan.router_mesh_port.ok_or_else(|| { miette::miette!("vm site `{}` is missing its router mesh port", plan.site_id) })?; @@ -4777,11 +5072,11 @@ fn project_dynamic_vm_router_surface( ) } -fn dynamic_child_route_overlay_id(plan: &SiteActuatorPlan, child_id: u64) -> String { +fn dynamic_child_route_overlay_id(plan: &SiteControllerRuntimePlan, child_id: u64) -> String { format!("framework-child:{}:{child_id}", plan.site_id) } -fn site_router_control_endpoint(plan: &SiteActuatorPlan) -> Result { +fn site_router_control_endpoint(plan: &SiteControllerRuntimePlan) -> Result { let state_path = Path::new(&plan.site_state_root).join("manager-state.json"); if state_path.is_file() { let state: SiteManagerState = read_json(&state_path, "site manager state")?; @@ -4805,7 +5100,7 @@ fn site_router_control_endpoint(plan: &SiteActuatorPlan) -> Result Result<(Vec, Vec)> { @@ -4914,15 +5209,20 @@ fn child_router_overlay_payload( Ok((peers, inbound_routes)) } -fn child_overlay_runtime_root(plan: &SiteActuatorPlan, child: &SiteActuatorChildRecord) -> PathBuf { +fn child_overlay_runtime_root( + plan: &SiteControllerRuntimePlan, + child: &SiteControllerRuntimeChildRecord, +) -> PathBuf { match plan.kind { - SiteKind::Direct | SiteKind::Vm => site_actuator_child_runtime_root(plan, child.child_id), + SiteKind::Direct | SiteKind::Vm => { + site_controller_runtime_child_runtime_root(plan, child.child_id) + } SiteKind::Compose | SiteKind::Kubernetes => PathBuf::from(&child.artifact_root), } } fn dynamic_route_issuer_grants( - children: &[SiteActuatorChildRecord], + children: &[SiteControllerRuntimeChildRecord], ) -> Result>> { let mut issuers_by_route_id = BTreeMap::>::new(); for child in children { @@ -4939,8 +5239,8 @@ fn dynamic_route_issuer_grants( } fn load_published_component_peers( - plan: &SiteActuatorPlan, - published_children: &[SiteActuatorChildRecord], + plan: &SiteControllerRuntimePlan, + published_children: &[SiteControllerRuntimeChildRecord], ) -> Result> { let mut component_peers = BTreeMap::new(); for child in published_children { @@ -5022,7 +5322,7 @@ fn apply_dynamic_route_issuer_grants( Ok(()) } -async fn reconcile_dynamic_site_router_overlays(app: &SiteActuatorApp) -> Result<()> { +async fn reconcile_dynamic_site_router_overlays(app: &SiteControllerRuntimeApp) -> Result<()> { let published_children = { let state = app.state.lock().await; state @@ -5041,9 +5341,9 @@ async fn reconcile_dynamic_site_router_overlays(app: &SiteActuatorApp) -> Result } async fn reconcile_dynamic_site_router_overlays_for_children( - app: &SiteActuatorApp, - overlay_children: &[SiteActuatorChildRecord], - issuer_children: &[SiteActuatorChildRecord], + app: &SiteControllerRuntimeApp, + overlay_children: &[SiteControllerRuntimeChildRecord], + issuer_children: &[SiteControllerRuntimeChildRecord], ) -> Result<()> { if overlay_children.is_empty() { return Ok(()); @@ -5080,8 +5380,8 @@ async fn reconcile_dynamic_site_router_overlays_for_children( } async fn apply_dynamic_site_router_overlay( - plan: &SiteActuatorPlan, - child: &SiteActuatorChildRecord, + plan: &SiteControllerRuntimePlan, + child: &SiteControllerRuntimeChildRecord, ) -> Result<()> { let artifact_root = Path::new(&child.artifact_root); let runtime_root = child_overlay_runtime_root(plan, child); @@ -5101,8 +5401,8 @@ async fn apply_dynamic_site_router_overlay( } async fn revoke_dynamic_site_router_overlay( - plan: &SiteActuatorPlan, - child: &SiteActuatorChildRecord, + plan: &SiteControllerRuntimePlan, + child: &SiteControllerRuntimeChildRecord, ) -> Result<()> { let endpoint = site_router_control_endpoint(plan)?; revoke_route_overlay_with_retry( @@ -5113,12 +5413,12 @@ async fn revoke_dynamic_site_router_overlay( .await } -async fn actuator_prepare_child( - app: &SiteActuatorApp, +async fn site_controller_runtime_prepare_child( + app: &SiteControllerRuntimeApp, child_id: u64, site_plan: DynamicSitePlanRecord, ) -> Result<()> { - let artifact_root = site_actuator_child_artifact_root(&app.plan, child_id); + let artifact_root = site_controller_runtime_child_artifact_root(&app.plan, child_id); let published_children = { let state = app.state.lock().await; state @@ -5224,7 +5524,7 @@ async fn actuator_prepare_child( let mut state = app.state.lock().await; state.children.insert( child_id, - SiteActuatorChildRecord { + SiteControllerRuntimeChildRecord { child_id, artifact_root: artifact_root.display().to_string(), assigned_components: site_plan.assigned_components.clone(), @@ -5237,18 +5537,16 @@ async fn actuator_prepare_child( write_json(&app.state_path, &*state) } -async fn actuator_publish_child( - app: &SiteActuatorApp, +async fn site_controller_runtime_publish_child( + app: &SiteControllerRuntimeApp, child_id: u64, site_plan: DynamicSitePlanRecord, ) -> Result<()> { let (child, published_children) = { let state = app.state.lock().await; - let child = state - .children - .get(&child_id) - .cloned() - .ok_or_else(|| miette::miette!("site actuator child {child_id} is not prepared"))?; + let child = state.children.get(&child_id).cloned().ok_or_else(|| { + miette::miette!("site controller runtime child {child_id} is not prepared") + })?; let published_children = state .children .values() @@ -5268,11 +5566,11 @@ async fn actuator_publish_child( let existing_peer_identities = local_direct_peer_identities(&app.plan, &state)?; drop(state); write_json( - &site_actuator_child_peer_ports_path(&app.plan, child_id), + &site_controller_runtime_child_peer_ports_path(&app.plan, child_id), &existing_peer_ports, )?; write_json( - &site_actuator_child_peer_identities_path(&app.plan, child_id), + &site_controller_runtime_child_peer_identities_path(&app.plan, child_id), &existing_peer_identities, )?; project_dynamic_child_mesh_scope( @@ -5284,9 +5582,9 @@ async fn actuator_publish_child( if dynamic_route_overlay_path(Path::new(&child.artifact_root)).is_file() { apply_dynamic_site_router_overlay(&app.plan, &child).await?; } - let runtime_root = site_actuator_child_runtime_root(&app.plan, child_id); - let storage_root = site_actuator_child_storage_root(&app.plan, child_id); - let child_root = site_actuator_child_root(&app.plan, child_id); + let runtime_root = site_controller_runtime_child_runtime_root(&app.plan, child_id); + let storage_root = site_controller_runtime_child_storage_root(&app.plan, child_id); + let child_root = site_controller_runtime_child_root(&app.plan, child_id); fs::create_dir_all(&runtime_root) .into_diagnostic() .wrap_err_with(|| format!("failed to create {}", runtime_root.display()))?; @@ -5302,13 +5600,24 @@ async fn actuator_publish_child( .arg("--runtime-root") .arg(&runtime_root) .arg("--existing-peer-ports") - .arg(site_actuator_child_peer_ports_path(&app.plan, child_id)) + .arg(site_controller_runtime_child_peer_ports_path( + &app.plan, child_id, + )) .arg("--existing-peer-identities") - .arg(site_actuator_child_peer_identities_path( + .arg(site_controller_runtime_child_peer_identities_path( &app.plan, child_id, )) .arg("--skip-router"); })?; + { + let mut state = app.state.lock().await; + let record = state + .children + .get_mut(&child_id) + .expect("prepared child should remain present"); + record.process_pid = Some(process.id()); + write_json(&app.state_path, &*state)?; + } wait_for_detached_child_runtime_state( process.id(), &direct_runtime_state_path(Path::new(&child.artifact_root)), @@ -5353,11 +5662,11 @@ async fn actuator_publish_child( let existing_peer_identities = local_vm_peer_identities(&app.plan, &state)?; drop(state); write_json( - &site_actuator_child_peer_ports_path(&app.plan, child_id), + &site_controller_runtime_child_peer_ports_path(&app.plan, child_id), &existing_peer_ports, )?; write_json( - &site_actuator_child_peer_identities_path(&app.plan, child_id), + &site_controller_runtime_child_peer_identities_path(&app.plan, child_id), &existing_peer_identities, )?; project_dynamic_child_mesh_scope( @@ -5369,9 +5678,9 @@ async fn actuator_publish_child( if dynamic_route_overlay_path(Path::new(&child.artifact_root)).is_file() { apply_dynamic_site_router_overlay(&app.plan, &child).await?; } - let runtime_root = site_actuator_child_runtime_root(&app.plan, child_id); - let storage_root = site_actuator_child_storage_root(&app.plan, child_id); - let child_root = site_actuator_child_root(&app.plan, child_id); + let runtime_root = site_controller_runtime_child_runtime_root(&app.plan, child_id); + let storage_root = site_controller_runtime_child_storage_root(&app.plan, child_id); + let child_root = site_controller_runtime_child_root(&app.plan, child_id); fs::create_dir_all(&runtime_root) .into_diagnostic() .wrap_err_with(|| format!("failed to create {}", runtime_root.display()))?; @@ -5387,13 +5696,24 @@ async fn actuator_publish_child( .arg("--runtime-root") .arg(&runtime_root) .arg("--existing-peer-ports") - .arg(site_actuator_child_peer_ports_path(&app.plan, child_id)) + .arg(site_controller_runtime_child_peer_ports_path( + &app.plan, child_id, + )) .arg("--existing-peer-identities") - .arg(site_actuator_child_peer_identities_path( + .arg(site_controller_runtime_child_peer_identities_path( &app.plan, child_id, )) .arg("--skip-router"); })?; + { + let mut state = app.state.lock().await; + let record = state + .children + .get_mut(&child_id) + .expect("prepared child should remain present"); + record.process_pid = Some(process.id()); + write_json(&app.state_path, &*state)?; + } wait_for_detached_child_runtime_state( process.id(), &Path::new(&child.artifact_root) @@ -5583,7 +5903,10 @@ async fn actuator_publish_child( Ok(()) } -async fn actuator_rollback_child(app: &SiteActuatorApp, child_id: u64) -> Result<()> { +async fn site_controller_runtime_rollback_child( + app: &SiteControllerRuntimeApp, + child_id: u64, +) -> Result<()> { let child = { let mut state = app.state.lock().await; let removed = state.children.remove(&child_id); @@ -5596,11 +5919,11 @@ async fn actuator_rollback_child(app: &SiteActuatorApp, child_id: u64) -> Result if let Some(pid) = child.process_pid { terminate_pid(pid, site_ready_timeout_for_kind(app.plan.kind))?; } - remove_dir_if_exists(&site_actuator_child_root(&app.plan, child_id)) + remove_dir_if_exists(&site_controller_runtime_child_root(&app.plan, child_id)) } -async fn actuator_destroy_child( - app: &SiteActuatorApp, +async fn site_controller_runtime_destroy_child( + app: &SiteControllerRuntimeApp, child_id: u64, desired_site_plan: Option, ) -> Result<()> { @@ -5608,19 +5931,13 @@ async fn actuator_destroy_child( let state = app.state.lock().await; state.children.get(&child_id).cloned() }; - if child.as_ref().is_some_and(|child| child.published) + if let Some(child) = child.as_ref() && matches!( app.plan.kind, SiteKind::Direct | SiteKind::Vm | SiteKind::Compose | SiteKind::Kubernetes ) { - revoke_dynamic_site_router_overlay( - &app.plan, - child - .as_ref() - .expect("published child should be available for overlay revoke"), - ) - .await?; + revoke_dynamic_site_router_overlay(&app.plan, child).await?; } if let Some(pid) = child.as_ref().and_then(|child| child.process_pid) { terminate_pid(pid, site_ready_timeout_for_kind(app.plan.kind))?; @@ -5629,9 +5946,9 @@ async fn actuator_destroy_child( match app.plan.kind { SiteKind::Compose => { let _ = desired_site_plan; - let child = child - .as_ref() - .ok_or_else(|| miette::miette!("site actuator child {child_id} is not prepared"))?; + let child = child.as_ref().ok_or_else(|| { + miette::miette!("site controller runtime child {child_id} is not prepared") + })?; let metadata = load_dynamic_compose_child_metadata(Path::new(&child.artifact_root))?; let compose_project = app.plan.compose_project.as_deref().ok_or_else(|| { miette::miette!( @@ -5666,9 +5983,9 @@ async fn actuator_destroy_child( } SiteKind::Kubernetes => { let _ = desired_site_plan; - let child = child - .as_ref() - .ok_or_else(|| miette::miette!("site actuator child {child_id} is not prepared"))?; + let child = child.as_ref().ok_or_else(|| { + miette::miette!("site controller runtime child {child_id} is not prepared") + })?; let artifact_root = Path::new(&child.artifact_root); let files = read_artifact_snapshot(artifact_root)?; let destroy_bundle = project_kubernetes_dynamic_child_destroy_artifact_files(&files)?; @@ -5727,7 +6044,7 @@ async fn actuator_destroy_child( ) { reconcile_dynamic_site_router_overlays(app).await?; } - remove_dir_if_exists(&site_actuator_child_root(&app.plan, child_id)) + remove_dir_if_exists(&site_controller_runtime_child_root(&app.plan, child_id)) } fn replace_artifact_snapshot(root: &Path, files: &BTreeMap) -> Result<()> { @@ -5821,7 +6138,7 @@ fn wait_for_kubernetes_artifact_workloads_deleted( } async fn wait_for_kubernetes_site_router_ready( - plan: &SiteActuatorPlan, + plan: &SiteControllerRuntimePlan, timeout: Duration, ) -> Result<()> { debug_assert_eq!(plan.kind, SiteKind::Kubernetes); @@ -6218,8 +6535,8 @@ fn wait_for_detached_vm_child_endpoints_ready( } fn local_direct_peer_ports( - plan: &SiteActuatorPlan, - state: &SiteActuatorState, + plan: &SiteControllerRuntimePlan, + state: &SiteControllerRuntimeState, ) -> Result> { let children = state .children @@ -6231,8 +6548,8 @@ fn local_direct_peer_ports( } fn local_direct_peer_ports_for_children( - plan: &SiteActuatorPlan, - children: &[SiteActuatorChildRecord], + plan: &SiteControllerRuntimePlan, + children: &[SiteControllerRuntimeChildRecord], ) -> Result> { let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) @@ -6254,15 +6571,15 @@ fn local_direct_peer_ports_for_children( for child in children { peers.extend(direct_peer_ports_for_artifact( Path::new(&child.artifact_root), - &site_actuator_child_runtime_root(plan, child.child_id), + &site_controller_runtime_child_runtime_root(plan, child.child_id), )?); } Ok(peers) } fn local_direct_peer_identities( - plan: &SiteActuatorPlan, - state: &SiteActuatorState, + plan: &SiteControllerRuntimePlan, + state: &SiteControllerRuntimeState, ) -> Result> { let children = state .children @@ -6274,8 +6591,8 @@ fn local_direct_peer_identities( } fn local_direct_peer_identities_for_children( - plan: &SiteActuatorPlan, - children: &[SiteActuatorChildRecord], + plan: &SiteControllerRuntimePlan, + children: &[SiteControllerRuntimeChildRecord], ) -> Result> { let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) @@ -6297,7 +6614,7 @@ fn local_direct_peer_identities_for_children( for child in children { peers.extend(direct_peer_identities_for_artifact( Path::new(&child.artifact_root), - &site_actuator_child_runtime_root(plan, child.child_id), + &site_controller_runtime_child_runtime_root(plan, child.child_id), )?); } Ok(peers) @@ -6307,10 +6624,6 @@ fn direct_peer_ports_for_artifact( artifact_root: &Path, runtime_root: &Path, ) -> Result> { - let state: crate::direct_runtime::DirectRuntimeState = read_json( - &direct_runtime_state_path(artifact_root), - "direct runtime state", - )?; let plan: DirectPlan = read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; let mut peers = BTreeMap::new(); for component in &plan.components { @@ -6318,17 +6631,7 @@ fn direct_peer_ports_for_artifact( &runtime_root.join(&component.sidecar.mesh_config_path), "mesh config", )?; - let port = state - .component_mesh_port_by_id - .get(&component.id) - .copied() - .ok_or_else(|| { - miette::miette!( - "direct runtime state is missing mesh port for component {}", - component.moniker - ) - })?; - peers.insert(config.identity.id, port); + peers.insert(config.identity.id, config.mesh_listen.port()); } Ok(peers) } @@ -6363,8 +6666,8 @@ fn direct_router_identity_for_artifact( } fn local_vm_peer_ports( - plan: &SiteActuatorPlan, - state: &SiteActuatorState, + plan: &SiteControllerRuntimePlan, + state: &SiteControllerRuntimeState, ) -> Result> { let children = state .children @@ -6376,8 +6679,8 @@ fn local_vm_peer_ports( } fn local_vm_peer_ports_for_children( - plan: &SiteActuatorPlan, - children: &[SiteActuatorChildRecord], + plan: &SiteControllerRuntimePlan, + children: &[SiteControllerRuntimeChildRecord], ) -> Result> { let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) @@ -6396,15 +6699,15 @@ fn local_vm_peer_ports_for_children( for child in children { peers.extend(vm_peer_ports_for_artifact( Path::new(&child.artifact_root), - &site_actuator_child_runtime_root(plan, child.child_id), + &site_controller_runtime_child_runtime_root(plan, child.child_id), )?); } Ok(peers) } fn local_vm_peer_identities( - plan: &SiteActuatorPlan, - state: &SiteActuatorState, + plan: &SiteControllerRuntimePlan, + state: &SiteControllerRuntimeState, ) -> Result> { let children = state .children @@ -6416,8 +6719,8 @@ fn local_vm_peer_identities( } fn local_vm_peer_identities_for_children( - plan: &SiteActuatorPlan, - children: &[SiteActuatorChildRecord], + plan: &SiteControllerRuntimePlan, + children: &[SiteControllerRuntimeChildRecord], ) -> Result> { let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) @@ -6438,7 +6741,7 @@ fn local_vm_peer_identities_for_children( for child in children { peers.extend(vm_peer_identities_for_artifact( Path::new(&child.artifact_root), - &site_actuator_child_runtime_root(plan, child.child_id), + &site_controller_runtime_child_runtime_root(plan, child.child_id), )?); } Ok(peers) @@ -6643,7 +6946,7 @@ fn prepare_kubernetes_artifact_namespace( } fn site_supervisor_plan_for_actuator( - plan: &SiteActuatorPlan, + plan: &SiteControllerRuntimePlan, artifact_dir: &Path, kubernetes_namespace: Option, ) -> SiteSupervisorPlan { @@ -6667,14 +6970,14 @@ fn site_supervisor_plan_for_actuator( port_forward_mesh_port: None, port_forward_control_port: None, observability_endpoint: plan.observability_endpoint.clone(), - framework_ccs_plan_path: None, - site_actuator_plan_path: None, + site_controller_plan_path: None, + site_controller_url: None, launch_env: plan.launch_env.clone(), } } fn prepare_kubernetes_artifact_for_apply( - plan: &SiteActuatorPlan, + plan: &SiteControllerRuntimePlan, artifact_dir: &Path, ) -> Result { debug_assert_eq!(plan.kind, SiteKind::Kubernetes); @@ -6861,8 +7164,8 @@ fn site_supervisor_plan_path(site_state_root: &Path) -> PathBuf { site_state_root.join("site-supervisor-plan.json") } -fn site_actuator_plan_path(site_state_root: &Path) -> PathBuf { - site_state_root.join("site-actuator-plan.json") +pub(crate) fn site_controller_plan_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("site-controller-plan.json") } pub(crate) fn desired_links_path(site_state_root: &Path) -> PathBuf { diff --git a/cli/src/mixed_run/launch_bundle.rs b/cli/src/mixed_run/launch_bundle.rs index 975fb065..9bca52b0 100644 --- a/cli/src/mixed_run/launch_bundle.rs +++ b/cli/src/mixed_run/launch_bundle.rs @@ -53,8 +53,39 @@ pub(super) fn materialize_launch_bundle( let run_plan_path = run_plan_path(bundle_root); write_json(&run_plan_path, run_plan)?; - let framework_control_state = - materialize_framework_control_state(run_plan, &state_root, run_id)?; + let controller_auth_token = amber_site_controller::generate_framework_auth_token( + &run_plan.mesh_scope, + "site-controller", + ); + let dynamic_capability_signing_seed_b64 = amber_mesh::dynamic_caps::signing_seed_b64( + &amber_mesh::dynamic_caps::signing_key_from_seed( + amber_mesh::dynamic_caps::generate_dynamic_capability_signing_seed(), + ), + ); + let dynamic_caps_token_verify_key_b64 = amber_mesh::dynamic_caps::verify_key_b64( + &amber_mesh::dynamic_caps::signing_key_from_seed_b64(&dynamic_capability_signing_seed_b64) + .map_err(|err| { + miette::miette!("site controller dynamic capability signing seed is invalid: {err}") + })?, + ); + let controller_endpoints = run_plan + .sites + .iter() + .map(|(site_id, site_plan)| { + let port = reserve_loopback_port()?; + let listen_addr = host_service_bind_addr_for_consumer(site_plan.site.kind, port); + let host = match site_plan.site.kind { + SiteKind::Direct | SiteKind::Vm => "127.0.0.1".to_string(), + SiteKind::Compose | SiteKind::Kubernetes => { + host_service_host_for_consumer(site_plan.site.kind) + } + }; + Ok(( + site_id.clone(), + (listen_addr, format!("http://{host}:{port}")), + )) + }) + .collect::>>()?; let observability = materialize_observability(bundle_root, run_id, &run_plan.mesh_scope, observability)?; @@ -63,49 +94,28 @@ pub(super) fn materialize_launch_bundle( .map(|materialized| materialized.receipt.endpoint.as_str()); let mut sites = BTreeMap::new(); - for (site_id, site_plan) in &run_plan.sites { + for (site_index, (site_id, site_plan)) in run_plan.sites.iter().enumerate() { let artifact_dir = materialize_site_artifacts(&sites_root, site_id, site_plan)?; let site_state_root = state_root.join(site_id); + let controller_state_path = site_state_root.join("site-controller-state.json"); + let controller_plan_path = site_controller_plan_path(&site_state_root); + let (controller_listen_addr, controller_url) = controller_endpoints + .get(site_id) + .cloned() + .expect("controller endpoint should exist for every site"); let mut framework_env = BTreeMap::new(); - let framework_ccs_plan_path = if let Some(control_state) = framework_control_state.as_ref() - { - let port = reserve_loopback_port()?; - let listen_addr = - crate::framework_component::ccs_listen_addr_for_site(site_plan.site.kind, port); - framework_env.insert( - amber_mesh::FRAMEWORK_COMPONENT_CCS_URL_ENV.to_string(), - crate::framework_component::ccs_url_for_site(site_plan.site.kind, port), - ); - framework_env.insert( - amber_mesh::FRAMEWORK_COMPONENT_CCS_AUTH_TOKEN_ENV.to_string(), - control_state.router_auth_token.clone(), - ); - framework_env.insert( - amber_mesh::DYNAMIC_CAPS_CONTROL_URL_ENV.to_string(), - control_state.receipt.url.clone(), - ); - framework_env.insert( - amber_mesh::DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV.to_string(), - control_state.control_state_auth_token.clone(), - ); - framework_env.insert( - amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV.to_string(), - control_state.dynamic_caps_token_verify_key_b64.clone(), - ); - let plan_path = site_state_root.join("framework-ccs-plan.json"); - crate::framework_component::write_framework_ccs_plan( - &plan_path, - site_id, - &site_state_root, - listen_addr, - control_state.receipt.url.as_str(), - &control_state.router_auth_token, - &control_state.control_state_auth_token, - )?; - Some(plan_path) - } else { - None - }; + framework_env.insert( + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV.to_string(), + controller_url.clone(), + ); + framework_env.insert( + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV.to_string(), + controller_auth_token.clone(), + ); + framework_env.insert( + amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV.to_string(), + dynamic_caps_token_verify_key_b64.clone(), + ); patch_site_artifacts( &artifact_dir, run_id, @@ -114,16 +124,6 @@ pub(super) fn materialize_launch_bundle( &merge_env_maps(runtime_env, &framework_env), observability_endpoint, )?; - let site_actuator_plan_path = if framework_control_state.is_some() - && run_plan - .dynamic_enabled_sites - .iter() - .any(|active| active == site_id) - { - Some(site_actuator_plan_path(&site_state_root)) - } else { - None - }; let launch_env = launch_env( run_id, &run_plan.mesh_scope, @@ -142,37 +142,71 @@ pub(super) fn materialize_launch_bundle( artifact_dir: &artifact_dir, site_state_root: &site_state_root, observability_endpoint, - framework_ccs_plan_path: framework_ccs_plan_path.as_deref(), - site_actuator_plan_path: site_actuator_plan_path.as_deref(), + site_controller_plan_path: Some(controller_plan_path.as_path()), + site_controller_url: Some(controller_url.as_str()), }, launch_env.clone(), )?; - if let Some(site_actuator_plan_path) = site_actuator_plan_path.as_ref() { - write_site_actuator_plan( - site_actuator_plan_path, - &SiteActuatorPlan { - schema: SITE_ACTUATOR_PLAN_SCHEMA.to_string(), - version: SITE_ACTUATOR_PLAN_VERSION, - run_id: run_id.to_string(), - mesh_scope: run_plan.mesh_scope.clone(), - run_root: bundle_root.display().to_string(), - site_id: site_id.clone(), - kind: site_plan.site.kind, - router_identity_id: site_plan.router_identity_id.clone(), - artifact_dir: artifact_dir.display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr: SocketAddr::from(([127, 0, 0, 1], reserve_loopback_port()?)), - storage_root: base_supervisor_plan.storage_root.clone(), - runtime_root: base_supervisor_plan.runtime_root.clone(), - router_mesh_port: base_supervisor_plan.router_mesh_port, - compose_project: base_supervisor_plan.compose_project.clone(), - kubernetes_namespace: base_supervisor_plan.kubernetes_namespace.clone(), - context: base_supervisor_plan.context.clone(), - observability_endpoint: base_supervisor_plan.observability_endpoint.clone(), - launch_env: launch_env.clone(), - }, - )?; - } + let controller_state = amber_site_controller::build_site_controller_state( + run_id, + run_plan, + site_id, + site_index, + run_plan.sites.len(), + &dynamic_capability_signing_seed_b64, + )?; + amber_site_controller::write_control_state(&controller_state_path, &controller_state)?; + let peer_controllers = controller_endpoints + .iter() + .filter(|(peer_site_id, _)| *peer_site_id != site_id) + .map(|(peer_site_id, (peer_listen_addr, _peer_url))| { + ( + peer_site_id.clone(), + amber_site_controller::SiteControllerPeerPlan { + site_id: peer_site_id.clone(), + kind: run_plan + .sites + .get(peer_site_id) + .expect("peer site should exist in run plan") + .site + .kind, + // Peer controllers talk to the host-side controller process directly. + // The site-facing authority URL may use container-oriented hostnames such + // as `host.docker.internal`, which are correct for workloads inside a + // site but wrong for another host-side controller process. + authority_url: amber_site_controller::authority_url_for_listen_addr( + *peer_listen_addr, + ), + }, + ) + }) + .collect(); + amber_site_controller::write_site_controller_plan( + &controller_plan_path, + run_id, + &run_plan.mesh_scope, + site_id, + site_plan.site.kind, + controller_listen_addr, + &controller_url, + &site_plan.router_identity_id, + &controller_state_path, + bundle_root, + &state_root, + &site_state_root, + &artifact_dir, + &controller_auth_token, + &dynamic_caps_token_verify_key_b64, + peer_controllers, + base_supervisor_plan.storage_root.as_deref(), + base_supervisor_plan.runtime_root.as_deref(), + base_supervisor_plan.router_mesh_port, + base_supervisor_plan.compose_project.as_deref(), + base_supervisor_plan.kubernetes_namespace.as_deref(), + base_supervisor_plan.context.as_deref(), + base_supervisor_plan.observability_endpoint.as_deref(), + &launch_env, + )?; write_json( &site_supervisor_plan_path(&site_state_root), &base_supervisor_plan, @@ -212,62 +246,11 @@ pub(super) fn materialize_launch_bundle( Ok(MaterializedLaunchBundle { run_plan_path, - framework_control_state, observability, sites, }) } -fn materialize_framework_control_state( - run_plan: &RunPlan, - state_root: &Path, - run_id: &str, -) -> Result> { - let control_state = crate::framework_component::build_control_state(run_id, run_plan)?; - let root = state_root.join("framework-component"); - let state_path = root.join("control-state.json"); - crate::framework_component::write_control_state(&state_path, &control_state)?; - let router_auth_token = - crate::framework_component::generate_framework_auth_token(&run_plan.mesh_scope, "router"); - let control_state_auth_token = crate::framework_component::generate_framework_auth_token( - &run_plan.mesh_scope, - "control-state", - ); - - let listen_addr = SocketAddr::from(([127, 0, 0, 1], reserve_loopback_port()?)); - let plan_path = root.join("control-state-plan.json"); - crate::framework_component::write_control_state_service_plan( - &plan_path, - listen_addr, - &state_path, - state_root.parent().unwrap_or(state_root), - state_root, - &run_plan.mesh_scope, - &control_state_auth_token, - )?; - let dynamic_caps_token_verify_key_b64 = amber_mesh::dynamic_caps::verify_key_b64( - &amber_mesh::dynamic_caps::signing_key_from_seed_b64( - &control_state.dynamic_capability_signing_seed_b64, - ) - .map_err(|err| { - miette::miette!( - "framework control-state dynamic capability signing seed is invalid: {err}" - ) - })?, - ); - - Ok(Some(MaterializedFrameworkControlState { - plan_path, - receipt: FrameworkControlStateReceipt { - pid: 0, - url: crate::framework_component::control_state_service_url(listen_addr), - }, - router_auth_token, - control_state_auth_token, - dynamic_caps_token_verify_key_b64, - })) -} - pub(super) fn build_launch_bundle_manifest( run_id: &str, source_plan_path: Option<&Path>, @@ -516,10 +499,20 @@ pub(super) fn preview_external_slot_url( } pub(super) fn site_launch_commands(plan: &SiteSupervisorPlan) -> Result> { - let exe = env::current_exe() - .into_diagnostic() - .wrap_err("failed to resolve amber executable path")?; - Ok(match plan.kind { + let exe = super::amber_cli_executable()?; + let mut commands = Vec::new(); + if let Some(plan_path) = plan.site_controller_plan_path.as_deref() { + let controller = super::site_controller_command()?; + let mut argv = vec![controller.executable.display().to_string()]; + argv.extend(controller.prefix_args.iter().map(|arg| (*arg).to_string())); + argv.extend(["--plan".to_string(), plan_path.to_string()]); + commands.push(LaunchCommandPreview { + argv, + env: plan.launch_env.clone(), + current_dir: Some(plan.site_state_root.clone()), + }); + } + let site_commands = match plan.kind { SiteKind::Direct => { let mut argv = vec![ exe.display().to_string(), @@ -657,7 +650,9 @@ pub(super) fn site_launch_commands(plan: &SiteSupervisorPlan) -> Result, -) -> Result> { - let Some(framework_control_state) = framework_control_state else { - return Ok(None); - }; - - let mut child = spawn_detached_child( - run_root, - &run_root - .join("state") - .join("framework-component") - .join("control-state.log"), - |cmd| { - cmd.arg("run-framework-control-state") - .arg("--plan") - .arg(&framework_control_state.plan_path); - }, - )?; - let listen_addr = framework_control_state - .receipt - .url - .strip_prefix("http://") - .ok_or_else(|| { - miette::miette!( - "framework control-state receipt url `{}` is not an http:// socket address", - framework_control_state.receipt.url - ) - })? - .parse::() - .into_diagnostic() - .wrap_err("invalid framework control-state listen address")?; - let deadline = Instant::now() + Duration::from_secs(10); - while Instant::now() < deadline { - if let Some(status) = child.try_wait().into_diagnostic()? { - return Err(miette::miette!( - "framework control-state service exited before becoming ready with status {status}" - )); - } - if wait_for_http_response(listen_addr, Duration::from_millis(250)).is_ok() { - let mut receipt = framework_control_state.receipt.clone(); - receipt.pid = child.id(); - return Ok(Some(receipt)); - } - sleep(Duration::from_millis(100)).await; - } - Err(miette::miette!( - "timed out waiting for framework control-state service" - )) -} - pub(super) fn prepare_site_launch( site: &MaterializedSite, runtime_env: &BTreeMap, @@ -949,7 +890,6 @@ pub(crate) async fn run_run_plan_with_id( ], ); - let mut framework_control_state_receipt = None; let mut launched_by_site = BTreeMap::::new(); let mut started_site_receipts = BTreeMap::::new(); let mut supervisor_children = BTreeMap::::new(); @@ -1010,13 +950,6 @@ pub(crate) async fn run_run_plan_with_id( sleep(delay).await; } } - - framework_control_state_receipt = start_materialized_framework_control_state( - &run_root, - launch_bundle.framework_control_state.as_ref(), - ) - .await?; - write_commit_marker(&run_root)?; emit_manager_event( "amber.run.committed", @@ -1032,7 +965,6 @@ pub(crate) async fn run_run_plan_with_id( plan_path: launch_bundle.run_plan_path.display().to_string(), source_plan_path: source_plan_path.map(|path| path.display().to_string()), run_root: run_root.display().to_string(), - framework_control_state: framework_control_state_receipt.clone(), observability: observability_receipt.clone(), bridge_proxies: bridge_proxies .values() @@ -1100,6 +1032,8 @@ pub(crate) async fn run_run_plan_with_id( router_mesh_addr: receipt.router_mesh_addr.clone(), router_identity_id: receipt.router_identity_id.clone(), router_public_key_b64: receipt.router_public_key_b64.clone(), + site_controller_pid: receipt.site_controller_pid, + site_controller_url: receipt.site_controller_url.clone(), last_error: Some("coordinator cleanup after failed startup".to_string()), }, ); @@ -1110,12 +1044,6 @@ pub(crate) async fn run_run_plan_with_id( { send_sigterm(pid); } - if let Some(pid) = framework_control_state_receipt - .as_ref() - .map(|value| value.pid) - { - send_sigterm(pid); - } } result @@ -1216,10 +1144,6 @@ pub(crate) async fn stop_run(run_id: &str, storage_root_override: Option<&Path>) { send_sigterm(pid); } - if let Some(framework_control_state) = receipt.framework_control_state.as_ref() { - send_sigterm(framework_control_state.pid); - } - if let Some(observability) = receipt.observability.as_ref() && let Some(pid) = observability.sink_pid && !wait_for_pid_exit(pid, PROCESS_SHUTDOWN_GRACE_PERIOD).await @@ -1229,15 +1153,6 @@ pub(crate) async fn stop_run(run_id: &str, storage_root_override: Option<&Path>) PROCESS_SHUTDOWN_GRACE_PERIOD.as_secs() )); } - if let Some(framework_control_state) = receipt.framework_control_state.as_ref() - && !wait_for_pid_exit(framework_control_state.pid, PROCESS_SHUTDOWN_GRACE_PERIOD).await - { - shutdown_failures.push(format!( - "framework control-state service (pid {}) did not stop within {}s", - framework_control_state.pid, - PROCESS_SHUTDOWN_GRACE_PERIOD.as_secs() - )); - } for proxy in &receipt.bridge_proxies { send_sigterm(proxy.pid); } diff --git a/cli/src/mixed_run/supervisor.rs b/cli/src/mixed_run/supervisor.rs index 4e68502a..91c98c2d 100644 --- a/cli/src/mixed_run/supervisor.rs +++ b/cli/src/mixed_run/supervisor.rs @@ -256,20 +256,14 @@ pub(super) fn build_supervisor_plan( None }, observability_endpoint: input.observability_endpoint.map(ToOwned::to_owned), - framework_ccs_plan_path: input - .framework_ccs_plan_path - .map(|path| path.display().to_string()), - site_actuator_plan_path: input - .site_actuator_plan_path + site_controller_plan_path: input + .site_controller_plan_path .map(|path| path.display().to_string()), + site_controller_url: input.site_controller_url.map(ToOwned::to_owned), launch_env, }) } -pub(super) fn write_site_actuator_plan(path: &Path, plan: &SiteActuatorPlan) -> Result<()> { - write_json(path, plan) -} - pub(super) fn spawn_site_supervisor(site_state_root: &Path) -> Result { let child = spawn_detached_child( site_state_root, @@ -650,6 +644,8 @@ pub(super) fn launched_site_from_state( router_mesh_addr: state.router_mesh_addr.clone(), router_identity_id: state.router_identity_id.clone(), router_public_key_b64: state.router_public_key_b64.clone(), + site_controller_pid: state.site_controller_pid, + site_controller_url: state.site_controller_url.clone(), }, router_control, router_identity, @@ -701,31 +697,22 @@ pub(super) async fn ensure_site_running( ) -> Result<()> { reap_child(&mut runtime.site_process)?; reap_child(&mut runtime.port_forward)?; - reap_child(&mut runtime.framework_ccs)?; - reap_child(&mut runtime.site_actuator)?; - - if runtime.framework_ccs.is_none() - && let Some(plan_path) = plan.framework_ccs_plan_path.as_deref() - { - runtime.framework_ccs = Some(spawn_runtime_process( - &PathBuf::from(&plan.site_state_root), - "framework-ccs.log", - &plan.launch_env, - |cmd| { - cmd.arg("run-framework-ccs").arg("--plan").arg(plan_path); - }, - )?); - } + reap_child(&mut runtime.site_controller)?; - if runtime.site_actuator.is_none() - && let Some(plan_path) = plan.site_actuator_plan_path.as_deref() + if runtime.site_controller.is_none() + && let Some(plan_path) = plan.site_controller_plan_path.as_deref() { - runtime.site_actuator = Some(spawn_runtime_process( + let controller = super::site_controller_command()?; + runtime.site_controller = Some(spawn_runtime_process_with_executable( + &controller.executable, &PathBuf::from(&plan.site_state_root), - "site-actuator.log", + "site-controller.log", &plan.launch_env, |cmd| { - cmd.arg("run-site-actuator").arg("--plan").arg(plan_path); + for arg in &controller.prefix_args { + cmd.arg(arg); + } + cmd.arg("--plan").arg(plan_path); }, )?); } @@ -1133,8 +1120,7 @@ pub(super) async fn cleanup_site( ) -> Result<()> { reap_child(&mut runtime.site_process)?; reap_child(&mut runtime.port_forward)?; - reap_child(&mut runtime.framework_ccs)?; - reap_child(&mut runtime.site_actuator)?; + reap_child(&mut runtime.site_controller)?; if let Some(child) = runtime.site_process.as_mut() { stop_child(child).await?; @@ -1142,17 +1128,13 @@ pub(super) async fn cleanup_site( if let Some(child) = runtime.port_forward.as_mut() { stop_child(child).await?; } - if let Some(child) = runtime.framework_ccs.as_mut() { - stop_child(child).await?; - } - if let Some(child) = runtime.site_actuator.as_mut() { + if let Some(child) = runtime.site_controller.as_mut() { stop_child(child).await?; } runtime.site_process = None; runtime.site_started = false; runtime.port_forward = None; - runtime.framework_ccs = None; - runtime.site_actuator = None; + runtime.site_controller = None; match plan.kind { SiteKind::Compose => { @@ -1238,6 +1220,8 @@ pub(super) fn build_site_state( router_mesh_addr, router_identity_id, router_public_key_b64, + site_controller_pid: runtime.site_controller.as_ref().map(Child::id), + site_controller_url: plan.site_controller_url.clone(), last_error, } } @@ -1274,6 +1258,8 @@ pub(super) fn persist_site_state( router_mesh_addr: launched.receipt.router_mesh_addr.clone(), router_identity_id: launched.receipt.router_identity_id.clone(), router_public_key_b64: launched.receipt.router_public_key_b64.clone(), + site_controller_pid: launched.receipt.site_controller_pid, + site_controller_url: launched.receipt.site_controller_url.clone(), last_error, }, ) @@ -1639,9 +1625,7 @@ pub(crate) fn spawn_detached_child( #[cfg(unix)] use std::os::unix::process::CommandExt as _; - let exe = env::current_exe() - .into_diagnostic() - .wrap_err("failed to resolve amber executable path")?; + let exe = super::amber_cli_executable()?; let log = fs::File::create(log_path) .into_diagnostic() .wrap_err_with(|| format!("failed to create log {}", log_path.display()))?; @@ -1678,9 +1662,17 @@ pub(super) fn spawn_runtime_process( extra_env: &BTreeMap, build: impl FnOnce(&mut Command), ) -> Result { - let exe = env::current_exe() - .into_diagnostic() - .wrap_err("failed to resolve amber executable path")?; + let exe = super::amber_cli_executable()?; + spawn_runtime_process_with_executable(&exe, site_state_root, log_name, extra_env, build) +} + +pub(super) fn spawn_runtime_process_with_executable( + executable: &Path, + site_state_root: &Path, + log_name: &str, + extra_env: &BTreeMap, + build: impl FnOnce(&mut Command), +) -> Result { let log_path = site_state_root.join(log_name); let log = fs::File::create(&log_path) .into_diagnostic() @@ -1689,7 +1681,7 @@ pub(super) fn spawn_runtime_process( .try_clone() .into_diagnostic() .wrap_err("failed to clone site log")?; - let mut cmd = Command::new(exe); + let mut cmd = Command::new(executable); cmd.envs(extra_env); cmd.stdout(Stdio::from(log)); cmd.stderr(Stdio::from(log_err)); diff --git a/cli/src/mixed_run/tests.rs b/cli/src/mixed_run/tests.rs index ef932fe0..848dbd27 100644 --- a/cli/src/mixed_run/tests.rs +++ b/cli/src/mixed_run/tests.rs @@ -2,8 +2,14 @@ use std::{ collections::{BTreeMap, BTreeSet}, io::{Read as _, Write as _}, net::{TcpListener, TcpStream}, + sync::Mutex, }; +use amber_compiler::reporter::direct::{ + DIRECT_PLAN_VERSION, DirectComponentPlan, DirectPlan, DirectProgramExecutionPlan, + DirectProgramPlan, DirectRuntimeAddressPlan, DirectSidecarPlan, +}; +use amber_site_controller::DynamicProxyExportRecord; use base64::Engine as _; use opentelemetry_proto::tonic::{ collector::logs::v1::ExportLogsServiceRequest, @@ -16,7 +22,8 @@ use tempfile::tempdir; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; use super::*; -use crate::framework_component::DynamicProxyExportRecord; + +static EXECUTABLE_ENV_LOCK: Mutex<()> = Mutex::new(()); fn test_dynamic_proxy_export_record( component_id: usize, @@ -56,6 +63,8 @@ fn test_site_receipt( router_mesh_addr: router_mesh_addr.map(str::to_string), router_identity_id: None, router_public_key_b64: None, + site_controller_pid: None, + site_controller_url: None, } } @@ -127,17 +136,19 @@ fn test_site_state( router_mesh_addr: router_mesh_addr.map(str::to_string), router_identity_id: None, router_public_key_b64: None, + site_controller_pid: None, + site_controller_url: None, last_error: None, } } -fn test_site_actuator_plan( +fn test_site_controller_runtime_plan( kind: SiteKind, artifact_dir: &Path, site_state_root: &Path, -) -> SiteActuatorPlan { - SiteActuatorPlan { - schema: "amber.run.site_actuator_plan".to_string(), +) -> SiteControllerRuntimePlan { + SiteControllerRuntimePlan { + schema: "amber.run.site_controller_runtime_plan".to_string(), version: 1, run_id: "run-test".to_string(), mesh_scope: "test-scope".to_string(), @@ -283,7 +294,7 @@ fn site_router_control_endpoint_prefers_manager_state_for_local_sites() { ) .expect("manager state should be written"); - let endpoint = site_router_control_endpoint(&test_site_actuator_plan( + let endpoint = site_router_control_endpoint(&test_site_controller_runtime_plan( kind, &artifact_dir, &site_state_root, @@ -293,7 +304,7 @@ fn site_router_control_endpoint_prefers_manager_state_for_local_sites() { assert_eq!( endpoint.to_string(), "unix:///tmp/router-from-manager.sock", - "local site actuator should use the live manager-state endpoint for {kind:?}", + "local site controller runtime should use the live manager-state endpoint for {kind:?}", ); } } @@ -370,6 +381,81 @@ fn mesh_config_local_targets_ready_rejects_unreachable_http_inbound_routes() { ); } +#[test] +fn direct_peer_ports_for_artifact_reads_mesh_listen_ports_without_runtime_state() { + let temp = tempdir().expect("tempdir should be created"); + let artifact_root = temp.path().join("artifact"); + let runtime_root = temp.path().join("runtime"); + let mesh_config_rel = PathBuf::from("mesh/components/app/mesh-config.json"); + let mesh_config_path = runtime_root.join(&mesh_config_rel); + fs::create_dir_all( + mesh_config_path + .parent() + .expect("mesh config should have a parent directory"), + ) + .expect("mesh config directory should be created"); + fs::create_dir_all(artifact_root.join(".amber")).expect("artifact .amber dir should exist"); + write_json( + &mesh_config_path, + &MeshConfigPublic { + identity: MeshIdentityPublic { + id: "/app".to_string(), + public_key: [9; 32], + mesh_scope: None, + }, + mesh_listen: SocketAddr::from(([127, 0, 0, 1], 18081)), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: TransportConfig::NoiseIk {}, + }, + ) + .expect("component mesh config should be written"); + write_json( + &artifact_root.join("direct-plan.json"), + &DirectPlan { + version: DIRECT_PLAN_VERSION.to_string(), + mesh_provision_plan: "{}".to_string(), + startup_order: vec![7], + components: vec![DirectComponentPlan { + id: 7, + moniker: "/app".to_string(), + log_name: "app".to_string(), + source_dir: None, + depends_on: Vec::new(), + sidecar: DirectSidecarPlan { + log_name: "app-sidecar".to_string(), + mesh_port: 0, + mesh_config_path: mesh_config_rel.display().to_string(), + mesh_identity_path: "mesh/components/app/mesh-identity.json".to_string(), + env_passthrough: Vec::new(), + }, + program: DirectProgramPlan { + log_name: "app-program".to_string(), + work_dir: "work/components/app".to_string(), + storage_mounts: Vec::new(), + execution: DirectProgramExecutionPlan::Direct { + entrypoint: vec!["/bin/echo".to_string()], + env: BTreeMap::new(), + }, + }, + }], + runtime_addresses: DirectRuntimeAddressPlan { + slots_by_scope: BTreeMap::new(), + slot_items_by_scope: BTreeMap::new(), + }, + router: None, + }, + ) + .expect("direct plan should be written"); + + let ports = direct_peer_ports_for_artifact(&artifact_root, &runtime_root).expect("peer ports"); + assert_eq!(ports.get("/app"), Some(&18081)); +} + #[test] fn read_compose_launch_env_returns_saved_launch_env() { let temp = tempdir().expect("tempdir should be created"); @@ -397,8 +483,8 @@ fn read_compose_launch_env_returns_saved_launch_env() { port_forward_mesh_port: None, port_forward_control_port: None, observability_endpoint: None, - framework_ccs_plan_path: None, - site_actuator_plan_path: None, + site_controller_plan_path: None, + site_controller_url: None, launch_env: BTreeMap::from([ ("AMBER_CONFIG_TENANT".to_string(), "acme-local".to_string()), ( @@ -423,6 +509,112 @@ fn read_compose_launch_env_returns_saved_launch_env() { ); } +#[test] +fn site_launch_commands_include_site_controller_binary_when_present() { + let _guard = EXECUTABLE_ENV_LOCK + .lock() + .unwrap_or_else(|err| err.into_inner()); + let temp = tempdir().expect("tempdir should be created"); + let amber = temp.path().join("amber"); + let controller = temp.path().join("amber-site-controller"); + fs::write(&amber, "").expect("amber binary placeholder should be written"); + fs::write(&controller, "").expect("site controller binary placeholder should be written"); + let previous_amber = std::env::var_os("CARGO_BIN_EXE_amber"); + let previous_controller = std::env::var_os("CARGO_BIN_EXE_amber-site-controller"); + // This test serializes access to process-global environment mutation with + // `EXECUTABLE_ENV_LOCK`, so these temporary overrides do not race other tests. + unsafe { + std::env::set_var("CARGO_BIN_EXE_amber", &amber); + std::env::set_var("CARGO_BIN_EXE_amber-site-controller", &controller); + } + + let plan = SiteSupervisorPlan { + schema: SITE_PLAN_SCHEMA.to_string(), + version: SITE_PLAN_VERSION, + run_id: "run-123".to_string(), + mesh_scope: "test.scope".to_string(), + run_root: temp.path().join("run-root").display().to_string(), + coordinator_pid: 1, + site_id: "compose_local".to_string(), + kind: SiteKind::Compose, + artifact_dir: temp.path().join("artifact").display().to_string(), + site_state_root: temp.path().join("state").display().to_string(), + storage_root: None, + runtime_root: None, + router_mesh_port: None, + compose_project: Some("amber-test".to_string()), + kubernetes_namespace: None, + context: None, + port_forward_mesh_port: None, + port_forward_control_port: None, + observability_endpoint: None, + site_controller_plan_path: Some( + temp.path() + .join("state") + .join("site-controller-plan.json") + .display() + .to_string(), + ), + site_controller_url: Some("http://127.0.0.1:41000".to_string()), + launch_env: BTreeMap::from([("AMBER_TEST".to_string(), "1".to_string())]), + }; + + let commands = site_launch_commands(&plan).expect("launch commands should build"); + assert_eq!( + commands.len(), + 2, + "compose site should list controller and site commands" + ); + assert_eq!(commands[0].argv[0], controller.display().to_string()); + assert_eq!(commands[0].argv[1], "--plan"); + assert!( + commands[0].argv[2].ends_with("site-controller-plan.json"), + "controller command should point at the site controller plan" + ); + assert_eq!(commands[0].env, plan.launch_env); + assert_eq!(commands[1].argv[0], "docker"); + + match previous_amber { + Some(value) => unsafe { std::env::set_var("CARGO_BIN_EXE_amber", value) }, + None => unsafe { std::env::remove_var("CARGO_BIN_EXE_amber") }, + } + match previous_controller { + Some(value) => unsafe { std::env::set_var("CARGO_BIN_EXE_amber-site-controller", value) }, + None => unsafe { std::env::remove_var("CARGO_BIN_EXE_amber-site-controller") }, + } +} + +#[test] +fn site_controller_command_falls_back_to_amber_subcommand_without_standalone_binary() { + let _guard = EXECUTABLE_ENV_LOCK + .lock() + .unwrap_or_else(|err| err.into_inner()); + let temp = tempdir().expect("tempdir should be created"); + let amber = temp.path().join("amber"); + let current = temp.path().join("not-the-controller"); + fs::write(&amber, "").expect("amber binary placeholder should be written"); + let previous_amber = std::env::var_os("CARGO_BIN_EXE_amber"); + let previous_controller = std::env::var_os("CARGO_BIN_EXE_amber-site-controller"); + unsafe { + std::env::set_var("CARGO_BIN_EXE_amber", &amber); + std::env::remove_var("CARGO_BIN_EXE_amber-site-controller"); + } + + let command = site_controller_command_from(¤t) + .expect("site controller command should fall back to amber"); + assert_eq!(command.executable, amber); + assert_eq!(command.prefix_args, vec!["run-site-controller"]); + + match previous_amber { + Some(value) => unsafe { std::env::set_var("CARGO_BIN_EXE_amber", value) }, + None => unsafe { std::env::remove_var("CARGO_BIN_EXE_amber") }, + } + match previous_controller { + Some(value) => unsafe { std::env::set_var("CARGO_BIN_EXE_amber-site-controller", value) }, + None => unsafe { std::env::remove_var("CARGO_BIN_EXE_amber-site-controller") }, + } +} + #[test] fn rewrite_dynamic_proxy_metadata_updates_compose_x_amber_exports() { let temp = tempdir().expect("tempdir should be created"); @@ -623,7 +815,7 @@ fn prepare_dynamic_compose_child_artifact_keeps_only_child_owned_services() { protocol: MeshProtocol::Http, http_plugins: Vec::new(), target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CCS_URL".to_string(), + url_env: "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL".to_string(), optional: false, }, allowed_issuers: vec!["/compose_admin".to_string()], @@ -636,7 +828,7 @@ fn prepare_dynamic_compose_child_artifact_keeps_only_child_owned_services() { protocol: MeshProtocol::Http, http_plugins: Vec::new(), target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CCS_URL".to_string(), + url_env: "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL".to_string(), optional: false, }, allowed_issuers: vec!["/job/root".to_string()], @@ -767,6 +959,8 @@ x-amber: router_public_key_b64: Some( base64::engine::general_purpose::STANDARD.encode([7u8; 32]), ), + site_controller_pid: None, + site_controller_url: None, last_error: None, }, ) @@ -816,7 +1010,7 @@ x-amber: .expect("compose subnet assignment should be idempotent"); prepare_dynamic_compose_child_artifact( - &SiteActuatorPlan { + &SiteControllerRuntimePlan { schema: SITE_PLAN_SCHEMA.to_string(), version: SITE_PLAN_VERSION, run_id: "run-123".to_string(), @@ -1196,7 +1390,7 @@ x-amber: .expect("site compose artifact should be written"); prepare_dynamic_compose_child_artifact( - &SiteActuatorPlan { + &SiteControllerRuntimePlan { schema: SITE_PLAN_SCHEMA.to_string(), version: SITE_PLAN_VERSION, run_id: "run-123".to_string(), @@ -1346,7 +1540,7 @@ x-amber: #[test] fn dynamic_route_issuer_grants_include_component_provide_inputs() { - let issuers = dynamic_route_issuer_grants(&[SiteActuatorChildRecord { + let issuers = dynamic_route_issuer_grants(&[SiteControllerRuntimeChildRecord { child_id: 7, artifact_root: "/tmp/child".to_string(), assigned_components: vec!["/sibling".to_string()], @@ -1647,7 +1841,7 @@ fn filter_dynamic_mesh_provision_plan_keeps_only_child_owned_router_routes() { protocol: MeshProtocol::Http, http_plugins: Vec::new(), target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CCS_URL".to_string(), + url_env: "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL".to_string(), optional: false, }, allowed_issuers: vec!["/vm_admin".to_string()], @@ -1660,7 +1854,7 @@ fn filter_dynamic_mesh_provision_plan_keeps_only_child_owned_router_routes() { protocol: MeshProtocol::Http, http_plugins: Vec::new(), target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CCS_URL".to_string(), + url_env: "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL".to_string(), optional: false, }, allowed_issuers: vec!["/job-compose/vm_helper/root".to_string()], @@ -2023,7 +2217,7 @@ fn child_router_overlay_payload_synthesizes_dynamic_proxy_export_routes_for_dire protocol: MeshProtocol::Http, http_plugins: Vec::new(), target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CCS_URL".to_string(), + url_env: "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL".to_string(), optional: false, }, allowed_issuers: vec!["/job/root".to_string()], @@ -2057,7 +2251,7 @@ fn child_router_overlay_payload_synthesizes_dynamic_proxy_export_routes_for_dire protocol: MeshProtocol::Http, http_plugins: Vec::new(), target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CCS_URL".to_string(), + url_env: "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL".to_string(), optional: false, }, allowed_issuers: vec!["/job/root".to_string()], @@ -2112,9 +2306,9 @@ fn child_router_overlay_payload_synthesizes_dynamic_proxy_export_routes_for_dire .expect("mesh config should be written"); let (_peers, inbound_routes) = child_router_overlay_payload( - &SiteActuatorPlan { - schema: SITE_ACTUATOR_PLAN_SCHEMA.to_string(), - version: SITE_ACTUATOR_PLAN_VERSION, + &SiteControllerRuntimePlan { + schema: SITE_CONTROLLER_RUNTIME_PLAN_SCHEMA.to_string(), + version: SITE_CONTROLLER_RUNTIME_PLAN_VERSION, run_id: "run-123".to_string(), mesh_scope: "mesh-scope-test".to_string(), run_root: temp.path().display().to_string(), @@ -2314,7 +2508,7 @@ fn prepare_dynamic_kubernetes_child_artifact_keeps_router_overlay_local() { protocol: MeshProtocol::Http, http_plugins: Vec::new(), target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CCS_URL".to_string(), + url_env: "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL".to_string(), optional: false, }, allowed_issuers: vec!["/kind_admin".to_string()], @@ -2327,7 +2521,7 @@ fn prepare_dynamic_kubernetes_child_artifact_keeps_router_overlay_local() { protocol: MeshProtocol::Http, http_plugins: Vec::new(), target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CCS_URL".to_string(), + url_env: "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL".to_string(), optional: false, }, allowed_issuers: vec!["/job/root".to_string()], @@ -2488,14 +2682,16 @@ secretGenerator: router_public_key_b64: Some( base64::engine::general_purpose::STANDARD.encode([7u8; 32]), ), + site_controller_pid: None, + site_controller_url: None, last_error: None, }, ) .expect("site manager state should be written"); prepare_dynamic_kubernetes_child_artifact( - &SiteActuatorPlan { - schema: SITE_ACTUATOR_PLAN_SCHEMA.to_string(), + &SiteControllerRuntimePlan { + schema: SITE_CONTROLLER_RUNTIME_PLAN_SCHEMA.to_string(), version: SITE_PLAN_VERSION, run_id: "run-123".to_string(), mesh_scope: "mesh-scope-test".to_string(), @@ -2875,21 +3071,21 @@ metadata: fn cleanup_dynamic_site_children_removes_child_roots_and_clears_state() { let temp = tempdir().expect("tempdir should be created"); let site_state_root = temp.path().join("state").join("direct_local"); - let child_root = site_actuator_child_root_for_site(&site_state_root, 7); + let child_root = site_controller_runtime_child_root_for_site(&site_state_root, 7); fs::create_dir_all(child_root.join("artifact")).expect("child artifact dir should exist"); fs::write(child_root.join("artifact").join("marker.txt"), "marker") .expect("child marker should be written"); write_json( - &site_actuator_state_path(&site_state_root), - &SiteActuatorState { - schema: "amber.run.site_actuator_state".to_string(), + &site_controller_runtime_state_path(&site_state_root), + &SiteControllerRuntimeState { + schema: "amber.run.site_controller_runtime_state".to_string(), version: 1, run_id: "run-123".to_string(), site_id: "direct_local".to_string(), kind: SiteKind::Direct, children: BTreeMap::from([( 7, - SiteActuatorChildRecord { + SiteControllerRuntimeChildRecord { child_id: 7, artifact_root: child_root.join("artifact").display().to_string(), assigned_components: Vec::new(), @@ -2901,16 +3097,16 @@ fn cleanup_dynamic_site_children_removes_child_roots_and_clears_state() { )]), }, ) - .expect("site actuator state should be written"); + .expect("site controller runtime state should be written"); cleanup_dynamic_site_children(&site_state_root, SiteKind::Direct) .expect("dynamic site children should be cleaned"); - let state: SiteActuatorState = read_json( - &site_actuator_state_path(&site_state_root), - "site actuator state", + let state: SiteControllerRuntimeState = read_json( + &site_controller_runtime_state_path(&site_state_root), + "site controller runtime state", ) - .expect("site actuator state should be readable"); + .expect("site controller runtime state should be readable"); assert!(state.children.is_empty()); assert!(!child_root.exists()); } @@ -3194,6 +3390,8 @@ fn test_launched_site_with_kind(kind: SiteKind) -> LaunchedSite { router_control: None, router_identity_id: None, router_public_key_b64: None, + site_controller_pid: None, + site_controller_url: None, }, router_identity: MeshIdentityPublic { id: format!("/site/{kind:?}"), @@ -3262,8 +3460,8 @@ fn prepare_kubernetes_site_artifact_for_apply_rewrites_runtime_artifact_state() let mut launch_env = BTreeMap::new(); launch_env.insert("AMBER_TEST_VALUE".to_string(), "fresh".to_string()); - let plan = SiteActuatorPlan { - schema: SITE_ACTUATOR_PLAN_SCHEMA.to_string(), + let plan = SiteControllerRuntimePlan { + schema: SITE_CONTROLLER_RUNTIME_PLAN_SCHEMA.to_string(), version: SITE_PLAN_VERSION, run_id: "run-1234abcd".to_string(), mesh_scope: "mesh-scope-test".to_string(), @@ -3352,7 +3550,6 @@ fn maybe_resolve_proxy_run_target_resolves_run_id_and_prefers_live_state() { plan_path: run_plan_path(&run_root).display().to_string(), source_plan_path: None, run_root: run_root.display().to_string(), - framework_control_state: None, observability: None, bridge_proxies: Vec::new(), sites: BTreeMap::from([( @@ -3425,7 +3622,6 @@ fn maybe_resolve_proxy_run_target_requires_site_for_multi_site_run() { plan_path: run_plan_path(&run_root).display().to_string(), source_plan_path: None, run_root: run_root.display().to_string(), - framework_control_state: None, observability: None, bridge_proxies: Vec::new(), sites: BTreeMap::from([ @@ -3488,7 +3684,6 @@ async fn stop_run_forces_supervisor_shutdown_and_cleans_up() { plan_path: run_plan_path(&run_root).display().to_string(), source_plan_path: None, run_root: run_root.display().to_string(), - framework_control_state: None, observability: None, bridge_proxies: Vec::new(), sites: BTreeMap::from([( @@ -3673,9 +3868,9 @@ async fn wait_for_kubernetes_site_router_ready_waits_for_live_discovery() { let started = Instant::now(); wait_for_kubernetes_site_router_ready( - &SiteActuatorPlan { - schema: SITE_ACTUATOR_PLAN_SCHEMA.to_string(), - version: SITE_ACTUATOR_PLAN_VERSION, + &SiteControllerRuntimePlan { + schema: SITE_CONTROLLER_RUNTIME_PLAN_SCHEMA.to_string(), + version: SITE_CONTROLLER_RUNTIME_PLAN_VERSION, run_id: "run-test".to_string(), mesh_scope: "mesh.scope.test".to_string(), run_root: temp.path().display().to_string(), diff --git a/cli/src/tests.rs b/cli/src/tests.rs index 7afeb6d3..0a6bf2be 100644 --- a/cli/src/tests.rs +++ b/cli/src/tests.rs @@ -1087,10 +1087,10 @@ fn rewrite_loopback_url_for_slirp_gateway_rewrites_loopback_only() { #[cfg(target_os = "linux")] #[test] -fn rewrite_sidecar_env_passthrough_for_slirp_rewrites_only_dynamic_caps_control_url() { +fn rewrite_sidecar_env_passthrough_for_slirp_rewrites_only_controller_url() { assert_eq!( rewrite_sidecar_env_passthrough_for_slirp( - amber_mesh::DYNAMIC_CAPS_CONTROL_URL_ENV, + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, "http://127.0.0.1:25000/v1/control-state" ), "http://10.0.2.2:25000/v1/control-state" diff --git a/cli/tests/linux_vm.rs b/cli/tests/linux_vm.rs index e5fcfd02..0992966e 100644 --- a/cli/tests/linux_vm.rs +++ b/cli/tests/linux_vm.rs @@ -10,6 +10,7 @@ mod outputs_root_support; mod workspace_root_support; use std::{ + collections::BTreeMap, env, fs, hash::{Hash as _, Hasher as _}, net::{SocketAddr, TcpListener}, @@ -75,7 +76,7 @@ impl ProvisionProfile { guest_packages: "build-essential ca-certificates curl git jq pkg-config libssl-dev bubblewrap \ slirp4netns docker.io", profile_setup: format!( - "if ! docker compose version >/dev/null 2>&1; then\n\ + "if ! docker compose version >/dev/null 2>&1; then\n\ sudo apt-get install -y --no-install-recommends docker-compose-v2 || sudo apt-get install -y --no-install-recommends docker-compose-plugin\n\ fi\n\ if ! docker buildx version >/dev/null 2>&1; then\n\ @@ -710,7 +711,8 @@ impl LinuxVmHarness { self.run_guest_checked( "prepare linux guest snapshot", "set -euxo pipefail\n. \"$HOME/.cargo/env\" || true\nsudo cloud-init clean --logs \ - --machine-id\nsudo rm -rf \"$HOME/amber\" \"$HOME/amber-target\"\nsudo sync\n", + --machine-id\nsudo rm -rf \"$HOME/amber\" \"$HOME/amber-target\" \ + \"$HOME/linux-vm-test-logs\"\nsudo sync\n", ) } @@ -767,8 +769,23 @@ impl LinuxVmHarness { ) } - fn run_test_command(&self, label: &str, command: &str) -> Result<(), String> { + fn run_test_command( + &self, + label: &str, + command: &str, + extra_env: &BTreeMap, + ) -> Result<(), String> { let arch = guest_arch(); + let extra_exports = extra_env + .iter() + .map(|(name, value)| format!("export {name}={}", shell_escape(value))) + .collect::>() + .join("\n"); + let extra_exports = if extra_exports.is_empty() { + String::new() + } else { + format!("{extra_exports}\n") + }; self.run_guest_checked( label, &format!( @@ -776,9 +793,22 @@ impl LinuxVmHarness { AMBER_TEST_KEEP_OUTPUTS=1\nexport \ AMBER_VM_SMOKE_BASE_IMAGE=\"$HOME/{image_filename}\"\nexport \ AMBER_MIXED_RUN_BASE_IMAGE=\"$HOME/{image_filename}\"\nexport \ - CARGO_TARGET_DIR=\"$HOME/amber-target\"\n{command}\n", + CARGO_TARGET_DIR=\"$HOME/amber-target\"\nexport CARGO_TERM_QUIET=true\nexport \ + CARGO_TERM_PROGRESS_WHEN=never\n{extra_exports}log_dir=\"$HOME/\ + linux-vm-test-logs\"\nmkdir -p \ + \"$log_dir\"\nstdout_log=\"$log_dir/{label}.stdout.log\"\nstderr_log=\"$log_dir/\ + {label}.stderr.log\"\nif {command} >\"$stdout_log\" 2>\"$stderr_log\"; then\ncat \ + \"$stdout_log\"\nelse\nstatus=$?\necho \"===== guest test panic lines \ + =====\"\ngrep -nE \"panicked at|assertion.*failed|thread '.*' panicked\" \ + \"$stdout_log\" \"$stderr_log\" || true\necho \"===== guest test kind load lines \ + =====\"\ngrep -n \"kind load docker-image\" \"$stdout_log\" \"$stderr_log\" || \ + true\necho \"===== guest test stdout tail ($stdout_log) =====\"\ntail -n 300 \ + \"$stdout_log\" || true\necho \"===== guest test stderr tail ($stderr_log) \ + =====\"\ntail -n 500 \"$stderr_log\" || true\nexit \"$status\"\nfi\n", guest = shell_escape(&self.guest_workspace), image_filename = arch.cloud_image_filename, + extra_exports = extra_exports, + label = label, command = command, ), ) @@ -795,11 +825,21 @@ impl LinuxVmHarness { $latest\"\nfind \"$latest\" -maxdepth 3 -type f | sort\nfor state_root in \ \"$latest/state\" \"$latest/replay-state\"; do\nif [ ! -d \"$state_root\" ]; \ then\ncontinue\nfi\necho \"===== state root: $state_root =====\"\nwhile IFS= read -r \ - path; do\necho \"----- $path -----\"\nsed -n '1,220p' \"$path\"\ndone < <(find \ + path; do\necho \"----- $path -----\"\nsed -n '1,260p' \"$path\"\ndone < <(find \ \"$state_root\" -type f \\( -name 'manager-state.json' -o -name \ 'direct-runtime-state.json' -o -name 'vm-runtime-state.json' -o -name \ + 'site-controller-state.json' -o -name 'site-controller-runtime-state.json' -o -name \ 'supervisor.log' -o -name 'port-forward.log' -o -name 'site.log' -o -name \ - 'control-state.log' \\) 2>/dev/null | sort)\ndone\n" + 'site-controller.log' -o -name 'site-controller-plan.json' -o -name \ + 'site-controller-runtime-plan.json' \\) 2>/dev/null | \ + sort)\ndone\nruns_root=\"$(find {guest_workspace} -path '*/.amber-runs/runs/run-*' \ + -type d | sort | tail -n 1)\"\nif [ -n \"$runs_root\" ]; then\necho \"latest guest \ + run root: $runs_root\"\nfind \"$runs_root/state\" -maxdepth 2 -type f \\( -name \ + 'manager-state.json' -o -name 'supervisor.log' -o -name 'port-forward.log' -o -name \ + 'site.log' -o -name 'site-controller.log' -o -name 'site-controller-state.json' -o \ + -name 'site-controller-runtime-state.json' -o -name 'site-controller-plan.json' -o \ + -name 'site-controller-runtime-plan.json' \\) | sort | while IFS= read -r path; \ + do\necho \"----- $path -----\"\nsed -n '1,260p' \"$path\"\ndone\nfi\n" ); self.ssh_output(&script) .ok() @@ -850,7 +890,7 @@ fn run_linux_guest_test( let provisioned_image = ensure_provisioned_image(profile)?; let harness = LinuxVmHarness::start(output_dir.path(), &provisioned_image)?; harness.copy_workspace()?; - harness.run_test_command(prefix, command) + harness.run_test_command(prefix, command, &BTreeMap::new()) })(); if result.is_err() || env::var_os("AMBER_TEST_KEEP_OUTPUTS").is_some() { output_dir.preserve(); @@ -864,6 +904,8 @@ fn run_linux_guest_test( fn ensure_provisioned_image(profile: ProvisionProfile) -> Result { let cache_path = provisioned_cache_path(profile); + // Keep only the provisioned base image as a persistent cache. It saves the expensive OS/tool + // setup without accumulating workspace-dependent snapshots or large build trees under target/. if cache_path.is_file() && env::var_os("AMBER_LINUX_VM_REFRESH_CACHE").is_none() { return Ok(cache_path); } @@ -974,6 +1016,72 @@ fn linux_vm_runs_framework_component_live_tests() { run_linux_guest_mixed_run_filter("framework_component_").unwrap_or_else(|err| panic!("{err}")); } +#[test] +#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component bounded \ + template replay live test inside the guest"] +fn linux_vm_runs_framework_component_bounded_template_frozen_source_replay_live() { + run_linux_guest_mixed_run_test( + "framework_component_bounded_template_frozen_source_replay_live", + ) + .unwrap_or_else(|err| panic!("{err}")); +} + +#[test] +#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component \ + cross-backend matrix live test inside the guest"] +fn linux_vm_runs_framework_component_cross_backend_matrix_live() { + run_linux_guest_mixed_run_test("framework_component_cross_backend_matrix_live") + .unwrap_or_else(|err| panic!("{err}")); +} + +#[test] +#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component \ + delegated-realm cross-site live test inside the guest"] +fn linux_vm_runs_framework_component_delegated_realm_cross_site_live() { + run_linux_guest_mixed_run_test("framework_component_delegated_realm_cross_site_live") + .unwrap_or_else(|err| panic!("{err}")); +} + +#[test] +#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component \ + direct-parent compose-child live test inside the guest"] +fn linux_vm_runs_framework_component_direct_parent_compose_child_live() { + run_linux_guest_mixed_run_test("framework_component_direct_parent_compose_child_live") + .unwrap_or_else(|err| panic!("{err}")); +} + +#[test] +#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component \ + dynamic-child teardown live test inside the guest"] +fn linux_vm_runs_framework_component_dynamic_children_teardown_with_run_live() { + run_linux_guest_mixed_run_test("framework_component_dynamic_children_teardown_with_run_live") + .unwrap_or_else(|err| panic!("{err}")); +} + +#[test] +#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component kind \ + creator after compose churn live test inside the guest"] +fn linux_vm_runs_framework_component_kind_creator_after_compose_churn_live() { + run_linux_guest_mixed_run_test("framework_component_kind_creator_after_compose_churn_live") + .unwrap_or_else(|err| panic!("{err}")); +} + +#[test] +#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component kind \ + creator compose-child live test inside the guest"] +fn linux_vm_runs_framework_component_kind_creator_compose_child_live() { + run_linux_guest_mixed_run_test("framework_component_kind_creator_compose_child_live") + .unwrap_or_else(|err| panic!("{err}")); +} + +#[test] +#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component kind \ + root-export live test inside the guest"] +fn linux_vm_runs_framework_component_kind_root_export_live() { + run_linux_guest_mixed_run_test("framework_component_kind_root_export_live") + .unwrap_or_else(|err| panic!("{err}")); +} + #[test] #[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux dynamic-capability live tests \ inside the guest"] diff --git a/cli/tests/mixed_run/smoke.rs b/cli/tests/mixed_run/smoke.rs index 1dc9a483..9f5d6338 100644 --- a/cli/tests/mixed_run/smoke.rs +++ b/cli/tests/mixed_run/smoke.rs @@ -739,27 +739,82 @@ fn write_dynamic_caps_component( } fn framework_control_state_path(run: &RunHandle) -> PathBuf { - run.run_root - .join("state") - .join("framework-component") - .join("control-state.json") + run.run_root.join("state") } -fn framework_control_state_plan_path(run: &RunHandle) -> PathBuf { - run.run_root - .join("state") - .join("framework-component") - .join("control-state-plan.json") +fn read_framework_control_state(control_state_root: &Path) -> Value { + let mut live_children = Vec::new(); + let entries = fs::read_dir(control_state_root) + .unwrap_or_else(|err| panic!("failed to read {}: {err}", control_state_root.display())); + for entry in entries.filter_map(Result::ok) { + let Ok(file_type) = entry.file_type() else { + continue; + }; + if !file_type.is_dir() { + continue; + } + let state_path = entry.path().join("site-controller-state.json"); + if !state_path.is_file() { + continue; + } + let state = read_json(&state_path); + if let Some(children) = state["live_children"].as_array() { + live_children.extend(children.iter().cloned()); + } + } + json!({ "live_children": live_children }) +} + +fn framework_site_controller_plan_path(run: &RunHandle) -> PathBuf { + let state_root = run.run_root.join("state"); + let entries = fs::read_dir(&state_root).expect("run state directory should be readable"); + let mut site_ids = entries + .filter_map(|entry| entry.ok()) + .filter_map(|entry| { + entry + .file_type() + .ok() + .filter(|file_type| file_type.is_dir()) + .map(|_| entry.file_name().to_string_lossy().to_string()) + }) + .collect::>(); + site_ids.sort(); + site_ids + .into_iter() + .map(|site_id| state_root.join(site_id).join("site-controller-plan.json")) + .find(|path| path.is_file()) + .expect("run should materialize at least one site controller plan") +} + +fn framework_site_controller_state_path(run: &RunHandle) -> PathBuf { + let state_root = run.run_root.join("state"); + let entries = fs::read_dir(&state_root).expect("run state directory should be readable"); + let mut site_ids = entries + .filter_map(|entry| entry.ok()) + .filter_map(|entry| { + entry + .file_type() + .ok() + .filter(|file_type| file_type.is_dir()) + .map(|_| entry.file_name().to_string_lossy().to_string()) + }) + .collect::>(); + site_ids.sort(); + site_ids + .into_iter() + .map(|site_id| state_root.join(site_id).join("site-controller-state.json")) + .find(|path| path.is_file()) + .expect("run should materialize at least one site controller state file") } -fn framework_control_state_post(run: &RunHandle, path: &str, payload: &Value) -> (u16, String) { - let plan = read_json(&framework_control_state_plan_path(run)); - let listen_addr = plan["listen_addr"] +fn framework_controller_post(run: &RunHandle, path: &str, payload: &Value) -> (u16, String) { + let plan = read_json(&framework_site_controller_plan_path(run)); + let authority_url = plan["authority_url"] .as_str() - .expect("framework control-state plan should publish listen_addr"); + .expect("site controller plan should publish authority_url"); let auth_token = plan["auth_token"] .as_str() - .expect("framework control-state plan should publish auth token"); + .expect("site controller plan should publish auth token"); let body = serde_json::to_string(payload).expect("request body should serialize"); let output = std::process::Command::new("curl") .arg("-sS") @@ -777,9 +832,9 @@ fn framework_control_state_post(run: &RunHandle, path: &str, payload: &Value) -> .arg("-") .arg("-w") .arg("\n%{http_code}") - .arg(format!("http://{listen_addr}{path}")) + .arg(format!("{}{path}", authority_url.trim_end_matches('/'))) .output() - .expect("framework control-state request should complete"); + .expect("site controller request should complete"); let stdout = String::from_utf8_lossy(&output.stdout); let (body, status) = stdout .rsplit_once('\n') @@ -797,7 +852,7 @@ fn wait_for_live_child(control_state_path: &Path, name: &str) -> u64 { wait_for_condition( Duration::from_secs(60), || { - read_json(control_state_path)["live_children"] + read_framework_control_state(control_state_path)["live_children"] .as_array() .is_some_and(|children| { children @@ -807,7 +862,7 @@ fn wait_for_live_child(control_state_path: &Path, name: &str) -> u64 { }, &format!("dynamic child `{name}` live in control state"), ); - let control_state = read_json(control_state_path); + let control_state = read_framework_control_state(control_state_path); control_state["live_children"] .as_array() .expect("live children should be an array") @@ -876,7 +931,7 @@ fn wait_for_framework_child_absent( wait_for_condition( timeout, || { - let no_live_child = read_json(control_state_path)["live_children"] + let no_live_child = read_framework_control_state(control_state_path)["live_children"] .as_array() .is_some_and(|children| children.iter().all(|child| child["name"] != child_name)); no_live_child && child_roots.iter().all(|root| !root.exists()) @@ -1645,7 +1700,7 @@ fn framework_component_direct_create_destroy_live() { Duration::from_secs(60), || { !child_root.exists() - && read_json(&control_state_path)["live_children"] + && read_framework_control_state(&control_state_path)["live_children"] .as_array() .is_some_and(|children| children.iter().all(|child| child["name"] != "job-1")) }, @@ -3623,7 +3678,7 @@ fn framework_component_destroy_of_provider_keeps_consumer_live() { Duration::from_secs(60), || { !source_root.exists() - && read_json(&control_state_path)["live_children"] + && read_framework_control_state(&control_state_path)["live_children"] .as_array() .is_some_and(|children| children.iter().all(|child| child["name"] != "source")) }, @@ -5817,7 +5872,8 @@ fn dynamic_capabilities_snapshot_replay_dynamic_child_live() { wait_for_condition( Duration::from_secs(60), || { - read_json(&replay_control_state)["base_scenario"]["components"] + read_json(&framework_site_controller_state_path(&replay_run))["base_scenario"] + ["components"] .as_array() .is_some_and(|components| { components @@ -5829,9 +5885,9 @@ fn dynamic_capabilities_snapshot_replay_dynamic_child_live() { ); wait_for_live_child(&replay_control_state, "job-replay"); let replay_held = response_json_from( - framework_control_state_post( + framework_controller_post( &replay_run, - "/v1/control-state/dynamic-caps/held", + "/v1/controller/dynamic-caps/held", &json!({ "holder_component_id": "components./job-replay" }), @@ -5871,9 +5927,9 @@ fn dynamic_capabilities_snapshot_replay_dynamic_child_live() { .expect("replay share should return a grant id"); let replay_ref_detail = response_json_from( - framework_control_state_post( + framework_controller_post( &replay_run, - "/v1/control-state/dynamic-caps/inspect-ref", + "/v1/controller/dynamic-caps/inspect-ref", &json!({ "holder_component_id": "components./job-replay", "ref": replay_ref @@ -5893,9 +5949,9 @@ fn dynamic_capabilities_snapshot_replay_dynamic_child_live() { .to_string(); let replay_held_detail = response_json_from( - framework_control_state_post( + framework_controller_post( &replay_run, - "/v1/control-state/dynamic-caps/held/detail", + "/v1/controller/dynamic-caps/held/detail", &json!({ "holder_component_id": "components./job-replay", "held_id": replay_shared_held_id @@ -5918,9 +5974,9 @@ fn dynamic_capabilities_snapshot_replay_dynamic_child_live() { "replay should either reuse the restored grant or create a new live one" ); - let (old_ref_status, old_ref_body) = framework_control_state_post( + let (old_ref_status, old_ref_body) = framework_controller_post( &replay_run, - "/v1/control-state/dynamic-caps/inspect-ref", + "/v1/controller/dynamic-caps/inspect-ref", &json!({ "holder_component_id": "components./job-replay", "ref": old_ref diff --git a/cli/tests/mixed_run/support.rs b/cli/tests/mixed_run/support.rs index 27130a78..3c6df9ae 100644 --- a/cli/tests/mixed_run/support.rs +++ b/cli/tests/mixed_run/support.rs @@ -630,22 +630,38 @@ pub(crate) fn ensure_docker_image(tag: &str, dockerfile: &Path) { return; } - let mut command = Command::new("docker"); - if docker_supports_buildx() { - command.arg("buildx").arg("build").arg("--load"); - } else { - command.env("DOCKER_BUILDKIT", "1"); - command.arg("build"); + if image_platform_opt(tag).is_some() { + return; } - let status = command - .arg("-t") - .arg(tag) - .arg("-f") - .arg(dockerfile) - .arg(workspace_root()) - .status() - .unwrap_or_else(|err| panic!("failed to build {tag}: {err}")); - assert!(status.success(), "docker build failed for {tag}"); + + let mut last_status = None; + for attempt in 1..=3 { + let mut command = Command::new("docker"); + if docker_supports_buildx() { + command.arg("buildx").arg("build").arg("--load"); + } else { + command.env("DOCKER_BUILDKIT", "1"); + command.arg("build"); + } + let status = command + .arg("-t") + .arg(tag) + .arg("-f") + .arg(dockerfile) + .arg(workspace_root()) + .status() + .unwrap_or_else(|err| panic!("failed to build {tag}: {err}")); + if status.success() { + return; + } + last_status = Some(status); + if attempt < 3 { + eprintln!("docker build attempt {attempt} failed for {tag}; retrying"); + thread::sleep(Duration::from_secs(attempt * 2)); + } + } + let status = last_status.expect("docker build should have produced a status"); + panic!("docker build failed for {tag} after retries with status {status}"); } pub(crate) fn docker_supports_buildx() -> bool { @@ -659,11 +675,10 @@ pub(crate) fn docker_supports_buildx() -> bool { }) } -pub(crate) fn ensure_internal_images() { +pub(crate) fn ensure_amber_internal_images() { static READY: OnceLock<()> = OnceLock::new(); READY.get_or_init(|| { let root = workspace_root(); - ensure_local_image(TEST_APP_IMAGE); ensure_docker_image( AMBER_ROUTER.reference, &root.join("docker/amber-router/Dockerfile"), @@ -679,21 +694,127 @@ pub(crate) fn ensure_internal_images() { }); } +pub(crate) fn ensure_internal_images() { + static READY: OnceLock<()> = OnceLock::new(); + READY.get_or_init(|| { + ensure_local_image(TEST_APP_IMAGE); + ensure_amber_internal_images(); + }); +} + pub(crate) fn load_kind_image(cluster_name: &str, image: &str) { - let status = Command::new("kind") + let mut last_error = None; + for attempt in 1..=3 { + let direct_error = match kind_load_docker_image(cluster_name, image) { + Ok(()) => return, + Err(err) => err, + }; + match kind_load_image_archive(cluster_name, image) { + Ok(()) => return, + Err(archive_error) => { + last_error = Some(format!( + "{direct_error}\n\nkind load image-archive fallback failed for {image} in \ + cluster {cluster_name}:\n{archive_error}" + )); + } + } + if attempt < 3 { + eprintln!( + "kind load docker-image attempt {attempt} failed for {image} in cluster \ + {cluster_name}; retrying" + ); + thread::sleep(Duration::from_secs(attempt * 2)); + } + } + + let error = last_error.expect("kind image load should record a failure"); + panic!("{error}"); +} + +fn kind_load_docker_image(cluster_name: &str, image: &str) -> Result<(), String> { + let output = Command::new("kind") .arg("load") .arg("docker-image") .arg("--name") .arg(cluster_name) .arg(image) - .status() - .unwrap_or_else(|err| { - panic!("failed to load {image} into kind cluster {cluster_name}: {err}") - }); - assert!( - status.success(), - "kind load docker-image failed for {image} in cluster {cluster_name}" - ); + .output() + .map_err(|err| { + format!("failed to run `kind load docker-image` for {image} in {cluster_name}: {err}") + })?; + if output.status.success() { + return Ok(()); + } + Err(format_command_output( + &format!("kind load docker-image failed for {image} in cluster {cluster_name}"), + &output, + )) +} + +fn kind_load_image_archive(cluster_name: &str, image: &str) -> Result<(), String> { + let mut docker = Command::new("docker") + .arg("image") + .arg("save") + .arg(image) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|err| { + format!("failed to run `docker image save` for {image} in {cluster_name}: {err}") + })?; + let docker_stdout = docker + .stdout + .take() + .ok_or_else(|| format!("docker image save for {image} did not expose stdout"))?; + let kind = Command::new("kind") + .arg("load") + .arg("image-archive") + .arg("--name") + .arg(cluster_name) + .arg("-") + .stdin(Stdio::from(docker_stdout)) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .map_err(|err| { + format!( + "failed to run `kind load image-archive` fallback for {image} in {cluster_name}: \ + {err}" + ) + })?; + let docker_output = docker.wait_with_output().map_err(|err| { + format!("failed to collect `docker image save` output for {image}: {err}") + })?; + if kind.status.success() && docker_output.status.success() { + return Ok(()); + } + + let mut message = String::new(); + if !docker_output.status.success() { + message.push_str(&format_command_output( + &format!("docker image save failed for {image}"), + &docker_output, + )); + } + if !kind.status.success() { + if !message.is_empty() { + message.push_str("\n\n"); + } + message.push_str(&format_command_output( + &format!("kind load image-archive failed for {image} in cluster {cluster_name}"), + &kind, + )); + } + Err(message) +} + +fn format_command_output(label: &str, output: &std::process::Output) -> String { + format!( + "{label} with status {}\nstdout:\n{}\nstderr:\n{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) } pub(crate) fn ensure_kind_internal_images(kind_cluster: &KindCluster) { @@ -710,7 +831,8 @@ pub(crate) fn ensure_kind_internal_images(kind_cluster: &KindCluster) { load_kind_image(&name, AMBER_ROUTER.reference); load_kind_image(&name, AMBER_PROVISIONER.reference); load_kind_image(&name, AMBER_HELPER.reference); - load_kind_image(&name, TEST_APP_IMAGE); + // The public test workload image is registry-pullable inside the kind cluster, so keep kind + // image preloading focused on the Amber images that only exist locally. loaded .lock() .expect("kind image-load guard should lock") diff --git a/compiler/src/targets/direct/mod.rs b/compiler/src/targets/direct/mod.rs index cd4a1181..fce2f68d 100644 --- a/compiler/src/targets/direct/mod.rs +++ b/compiler/src/targets/direct/mod.rs @@ -1087,7 +1087,9 @@ mod tests { }; use amber_manifest::{Manifest, ManifestRef}; - use amber_mesh::{FRAMEWORK_COMPONENT_CCS_AUTH_TOKEN_ENV, FRAMEWORK_COMPONENT_CCS_URL_ENV}; + use amber_mesh::{ + FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV, FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, + }; use amber_resolver::Resolver; use amber_scenario::{BindingEdge, Component, Moniker, Scenario}; use tempfile::TempDir; @@ -1171,7 +1173,7 @@ mod tests { } #[test] - fn direct_router_passthrough_includes_framework_ccs_auth() { + fn direct_router_passthrough_includes_framework_controller_auth() { let dir = TempDir::new().expect("temp dir"); let root_path = dir.path().join("root.json5"); let admin_path = dir.path().join("admin.json5"); @@ -1244,15 +1246,15 @@ mod tests { router .env_passthrough .iter() - .any(|env_var| env_var == FRAMEWORK_COMPONENT_CCS_URL_ENV), - "router must receive the framework CCS URL env passthrough", + .any(|env_var| env_var == FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV), + "router must receive the framework controller URL env passthrough", ); assert!( router .env_passthrough .iter() - .any(|env_var| env_var == FRAMEWORK_COMPONENT_CCS_AUTH_TOKEN_ENV), - "router must receive the framework CCS auth env passthrough", + .any(|env_var| env_var == FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV), + "router must receive the framework controller auth env passthrough", ); } diff --git a/compiler/src/targets/mesh/kubernetes/tests/rendering.rs b/compiler/src/targets/mesh/kubernetes/tests/rendering.rs index 8e746a88..1b32aa0a 100644 --- a/compiler/src/targets/mesh/kubernetes/tests/rendering.rs +++ b/compiler/src/targets/mesh/kubernetes/tests/rendering.rs @@ -770,11 +770,11 @@ fn kubernetes_templates_dynamic_caps_sidecar_control_env() { .get(&PathBuf::from(super::COMPONENT_SIDECAR_ENV_FILE)) .expect("component sidecar env template"); assert!( - sidecar_env.contains("AMBER_DYNAMIC_CAPS_CONTROL_URL="), + sidecar_env.contains("AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL="), "{sidecar_env}" ); assert!( - sidecar_env.contains("AMBER_DYNAMIC_CAPS_CONTROL_AUTH_TOKEN="), + sidecar_env.contains("AMBER_FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN="), "{sidecar_env}" ); assert!( diff --git a/compiler/src/targets/mesh/mesh_config.rs b/compiler/src/targets/mesh/mesh_config.rs index a19e7bfe..076402e0 100644 --- a/compiler/src/targets/mesh/mesh_config.rs +++ b/compiler/src/targets/mesh/mesh_config.rs @@ -2,9 +2,8 @@ use std::collections::{BTreeSet, HashMap}; use amber_manifest::NetworkProtocol; use amber_mesh::{ - DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV, DYNAMIC_CAPS_CONTROL_URL_ENV, - DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, FRAMEWORK_COMPONENT_CCS_AUTH_TOKEN_ENV, - FRAMEWORK_COMPONENT_CCS_URL_ENV, InboundRoute, InboundTarget, MeshConfigTemplate, + DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV, + FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, InboundRoute, InboundTarget, MeshConfigTemplate, MeshIdentityTemplate, MeshPeerTemplate, MeshProtocol, OutboundRoute, component_route_id, framework_cap_instance_id, http_route_plugins_for_capability_kind, router_export_route_id, router_external_route_id, telemetry::SCENARIO_RUN_ID_ENV, @@ -190,11 +189,11 @@ pub(crate) fn build_mesh_config_plan( push_env_passthrough_once(&mut component_sidecar_env_passthrough, SCENARIO_RUN_ID_ENV); push_env_passthrough_once( &mut component_sidecar_env_passthrough, - DYNAMIC_CAPS_CONTROL_URL_ENV, + FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, ); push_env_passthrough_once( &mut component_sidecar_env_passthrough, - DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV, + FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV, ); push_env_passthrough_once( &mut component_sidecar_env_passthrough, @@ -499,10 +498,13 @@ pub(crate) fn build_mesh_config_plan( .filter(|binding| binding.capability.as_str() == "component") .collect::>(); if !framework_bindings.is_empty() { - push_env_passthrough_once(&mut router_env_passthrough, FRAMEWORK_COMPONENT_CCS_URL_ENV); push_env_passthrough_once( &mut router_env_passthrough, - FRAMEWORK_COMPONENT_CCS_AUTH_TOKEN_ENV, + FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, + ); + push_env_passthrough_once( + &mut router_env_passthrough, + FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV, ); } for binding in framework_bindings { @@ -533,7 +535,7 @@ pub(crate) fn build_mesh_config_plan( protocol: MeshProtocol::Http, http_plugins: Vec::new(), target: InboundTarget::External { - url_env: FRAMEWORK_COMPONENT_CCS_URL_ENV.to_string(), + url_env: FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV.to_string(), optional: false, }, allowed_issuers: vec![consumer_id], diff --git a/docker/amber-cli/Dockerfile b/docker/amber-cli/Dockerfile index 56797d2d..5da6ddea 100644 --- a/docker/amber-cli/Dockerfile +++ b/docker/amber-cli/Dockerfile @@ -41,6 +41,7 @@ COPY compiler/manifest/Cargo.toml compiler/manifest/ COPY runtime/mesh/Cargo.toml runtime/mesh/ COPY runtime/proxy/Cargo.toml runtime/proxy/ COPY runtime/router/Cargo.toml runtime/router/ +COPY runtime/site-controller/Cargo.toml runtime/site-controller/ COPY compiler/resolver/Cargo.toml compiler/resolver/ COPY compiler/scenario/Cargo.toml compiler/scenario/ COPY compiler/template/Cargo.toml compiler/template/ @@ -48,8 +49,8 @@ COPY runtime/provisioner/Cargo.toml runtime/provisioner/ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ - mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/router/src compiler/resolver/src compiler/scenario/src compiler/template/src runtime/provisioner/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ + mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/router/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src runtime/provisioner/src && \ + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ cargo fetch --locked COPY cli ./cli @@ -60,6 +61,7 @@ COPY runtime/helper ./runtime/helper COPY runtime/mesh ./runtime/mesh COPY runtime/proxy ./runtime/proxy COPY runtime/router ./runtime/router +COPY runtime/site-controller ./runtime/site-controller COPY docker/images.json docker/images.json ARG BUILD_MODE=release @@ -69,19 +71,21 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/app/target,sharing=locked \ target=$(cat /tmp/rust-target) && \ if [ "$BUILD_MODE" = "release" ]; then \ - AMBER_BUILD_VERSION="${AMBER_BUILD_VERSION}" cargo build --locked --release -p amber-cli --target "${target}"; \ + AMBER_BUILD_VERSION="${AMBER_BUILD_VERSION}" cargo build --locked --release -p amber-cli -p amber-site-controller --target "${target}"; \ build_dir=release; \ else \ - AMBER_BUILD_VERSION="${AMBER_BUILD_VERSION}" cargo build --locked -p amber-cli --target "${target}"; \ + AMBER_BUILD_VERSION="${AMBER_BUILD_VERSION}" cargo build --locked -p amber-cli -p amber-site-controller --target "${target}"; \ build_dir=debug; \ fi && \ install -D -m 0755 /app/target/"${target}"/"${build_dir}"/amber /out/amber && \ + install -D -m 0755 /app/target/"${target}"/"${build_dir}"/amber-site-controller /out/amber-site-controller && \ install -d -m 0755 /out/workdir FROM gcr.io/distroless/static-debian13 AS runtime COPY --from=builder --chown=65532:65532 /out/workdir /app COPY --from=builder --chown=65532:65532 /out/amber /amber +COPY --from=builder --chown=65532:65532 /out/amber-site-controller /amber-site-controller USER 65532:65532 WORKDIR /app diff --git a/docker/amber-docker-gateway/Dockerfile b/docker/amber-docker-gateway/Dockerfile index da97e92e..a08f9988 100644 --- a/docker/amber-docker-gateway/Dockerfile +++ b/docker/amber-docker-gateway/Dockerfile @@ -41,14 +41,15 @@ COPY runtime/mesh/Cargo.toml runtime/mesh/ COPY runtime/proxy/Cargo.toml runtime/proxy/ COPY runtime/provisioner/Cargo.toml runtime/provisioner/ COPY runtime/router/Cargo.toml runtime/router/ +COPY runtime/site-controller/Cargo.toml runtime/site-controller/ COPY compiler/resolver/Cargo.toml compiler/resolver/ COPY compiler/scenario/Cargo.toml compiler/scenario/ COPY compiler/template/Cargo.toml compiler/template/ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ - mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/provisioner/src runtime/router/src compiler/resolver/src compiler/scenario/src compiler/template/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/provisioner/src/main.rs runtime/router/src/main.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs && \ + mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/provisioner/src runtime/router/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src && \ + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/provisioner/src/main.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs && \ cargo fetch --locked COPY runtime/docker-gateway ./runtime/docker-gateway diff --git a/docker/amber-helper/Dockerfile b/docker/amber-helper/Dockerfile index 53b69211..ea68f719 100644 --- a/docker/amber-helper/Dockerfile +++ b/docker/amber-helper/Dockerfile @@ -40,6 +40,7 @@ COPY compiler/manifest/Cargo.toml compiler/manifest/ COPY runtime/mesh/Cargo.toml runtime/mesh/ COPY runtime/proxy/Cargo.toml runtime/proxy/ COPY runtime/router/Cargo.toml runtime/router/ +COPY runtime/site-controller/Cargo.toml runtime/site-controller/ COPY compiler/resolver/Cargo.toml compiler/resolver/ COPY compiler/scenario/Cargo.toml compiler/scenario/ COPY compiler/template/Cargo.toml compiler/template/ @@ -47,8 +48,8 @@ COPY runtime/provisioner/Cargo.toml runtime/provisioner/ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ - mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/router/src compiler/resolver/src compiler/scenario/src compiler/template/src runtime/provisioner/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ + mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/router/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src runtime/provisioner/src && \ + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ cargo fetch --locked COPY compiler ./compiler diff --git a/docker/amber-manager/Dockerfile b/docker/amber-manager/Dockerfile index fbf2c9fe..af669fb8 100644 --- a/docker/amber-manager/Dockerfile +++ b/docker/amber-manager/Dockerfile @@ -41,14 +41,15 @@ COPY runtime/mesh/Cargo.toml runtime/mesh/ COPY runtime/proxy/Cargo.toml runtime/proxy/ COPY runtime/provisioner/Cargo.toml runtime/provisioner/ COPY runtime/router/Cargo.toml runtime/router/ +COPY runtime/site-controller/Cargo.toml runtime/site-controller/ COPY compiler/resolver/Cargo.toml compiler/resolver/ COPY compiler/scenario/Cargo.toml compiler/scenario/ COPY compiler/template/Cargo.toml compiler/template/ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ - mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/provisioner/src runtime/router/src compiler/resolver/src compiler/scenario/src compiler/template/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/provisioner/src/main.rs runtime/router/src/main.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs && \ + mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/provisioner/src runtime/router/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src && \ + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/provisioner/src/main.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs && \ cargo fetch --locked COPY compiler ./compiler diff --git a/docker/amber-provisioner/Dockerfile b/docker/amber-provisioner/Dockerfile index d7f58c55..61e9c35f 100644 --- a/docker/amber-provisioner/Dockerfile +++ b/docker/amber-provisioner/Dockerfile @@ -38,12 +38,13 @@ COPY runtime/mesh/Cargo.toml runtime/mesh/ COPY runtime/proxy/Cargo.toml runtime/proxy/ COPY runtime/provisioner/Cargo.toml runtime/provisioner/ COPY runtime/router/Cargo.toml runtime/router/ +COPY runtime/site-controller/Cargo.toml runtime/site-controller/ COPY compiler/resolver/Cargo.toml compiler/resolver/ COPY compiler/scenario/Cargo.toml compiler/scenario/ COPY compiler/template/Cargo.toml compiler/template/ -RUN mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/provisioner/src runtime/router/src compiler/resolver/src compiler/scenario/src compiler/template/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/provisioner/src/main.rs runtime/router/src/main.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs +RUN mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/provisioner/src runtime/router/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src && \ + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/provisioner/src/main.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs RUN cargo fetch --locked COPY compiler ./compiler diff --git a/docker/amber-router/Dockerfile b/docker/amber-router/Dockerfile index d896be07..b52d9717 100644 --- a/docker/amber-router/Dockerfile +++ b/docker/amber-router/Dockerfile @@ -40,6 +40,7 @@ COPY compiler/manifest/Cargo.toml compiler/manifest/ COPY runtime/mesh/Cargo.toml runtime/mesh/ COPY runtime/proxy/Cargo.toml runtime/proxy/ COPY runtime/router/Cargo.toml runtime/router/ +COPY runtime/site-controller/Cargo.toml runtime/site-controller/ COPY compiler/resolver/Cargo.toml compiler/resolver/ COPY compiler/scenario/Cargo.toml compiler/scenario/ COPY compiler/template/Cargo.toml compiler/template/ @@ -47,8 +48,8 @@ COPY runtime/provisioner/Cargo.toml runtime/provisioner/ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ - mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/router/src compiler/resolver/src compiler/scenario/src compiler/template/src runtime/provisioner/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ + mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/router/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src runtime/provisioner/src && \ + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ cargo fetch --locked COPY compiler ./compiler diff --git a/runtime/mesh/src/dynamic_caps.rs b/runtime/mesh/src/dynamic_caps.rs index 8ded1d06..0c4e0448 100644 --- a/runtime/mesh/src/dynamic_caps.rs +++ b/runtime/mesh/src/dynamic_caps.rs @@ -16,8 +16,9 @@ pub const DYNAMIC_CAPS_REF_HOST: &str = "ref"; pub const DYNAMIC_CAPS_REF_VERSION: u32 = 1; pub const DYNAMIC_CAPS_API_URL_ENV: &str = "AMBER_DYNAMIC_CAPS_API_URL"; -pub const DYNAMIC_CAPS_CONTROL_URL_ENV: &str = "AMBER_DYNAMIC_CAPS_CONTROL_URL"; -pub const DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV: &str = "AMBER_DYNAMIC_CAPS_CONTROL_AUTH_TOKEN"; +pub const FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV: &str = "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL"; +pub const FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV: &str = + "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN"; pub const DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV: &str = "AMBER_DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/runtime/mesh/src/lib.rs b/runtime/mesh/src/lib.rs index dfb54a9b..20144d0f 100644 --- a/runtime/mesh/src/lib.rs +++ b/runtime/mesh/src/lib.rs @@ -207,12 +207,10 @@ pub struct MeshConfigTemplate { pub const MESH_PROVISION_PLAN_VERSION: &str = "2"; pub const MESH_CONFIG_FILENAME: &str = "mesh-config.json"; pub const MESH_IDENTITY_FILENAME: &str = "mesh-identity.json"; -pub const FRAMEWORK_COMPONENT_CCS_URL_ENV: &str = "AMBER_FRAMEWORK_COMPONENT_CCS_URL"; -pub const FRAMEWORK_COMPONENT_CCS_AUTH_TOKEN_ENV: &str = "AMBER_FRAMEWORK_COMPONENT_CCS_AUTH_TOKEN"; +pub const FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV: &str = "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL"; +pub const FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV: &str = + "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN"; pub const DYNAMIC_CAPS_API_URL_ENV: &str = dynamic_caps::DYNAMIC_CAPS_API_URL_ENV; -pub const DYNAMIC_CAPS_CONTROL_URL_ENV: &str = dynamic_caps::DYNAMIC_CAPS_CONTROL_URL_ENV; -pub const DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV: &str = - dynamic_caps::DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV; pub const DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV: &str = dynamic_caps::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV; diff --git a/runtime/proxy/src/lib.rs b/runtime/proxy/src/lib.rs index ec436977..c2dad81d 100644 --- a/runtime/proxy/src/lib.rs +++ b/runtime/proxy/src/lib.rs @@ -1736,6 +1736,13 @@ fn control_status(code: u16, body: &str) -> Result<(), ControlUpdateError> { }) } +fn control_delete_status(code: u16, body: &str) -> Result<(), ControlUpdateError> { + if (200..300).contains(&code) || code == 404 { + return Ok(()); + } + control_status(code, body) +} + async fn send_control_put_json( endpoint: &ControlEndpoint, path: &str, @@ -1775,7 +1782,7 @@ async fn send_control_delete_json( ); let response = send_control_request(endpoint, &request).await?; let (code, body) = parse_http_response(&response).ok_or(ControlUpdateError::Retryable)?; - control_status(code, body) + control_delete_status(code, body) } fn parse_http_response(response: &str) -> Option<(u16, &str)> { diff --git a/runtime/router/src/control.rs b/runtime/router/src/control.rs index 89272a24..6b3d2478 100644 --- a/runtime/router/src/control.rs +++ b/runtime/router/src/control.rs @@ -428,9 +428,11 @@ pub(super) async fn handle_inbound( optional, } => match route.protocol { MeshProtocol::Http => { - let framework_route_id = (url_env == amber_mesh::FRAMEWORK_COMPONENT_CCS_URL_ENV) + let framework_route_id = (url_env + == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV) .then(|| Arc::::from(route.route_id.as_str())); - let framework_peer_id = (url_env == amber_mesh::FRAMEWORK_COMPONENT_CCS_URL_ENV) + let framework_peer_id = (url_env + == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV) .then(|| Arc::::from(remote_id.as_str())); proxy_noise_to_external( &mut session, @@ -1320,10 +1322,18 @@ pub(super) async fn apply_route_overlay( let mut issuers = dynamic_issuers.write().await; remove_dynamic_issuer_grants(&mut issuers, &existing_overlay.static_issuer_grants); for peer in &existing_overlay.peers { - trust - .remove_peer(peer) - .await - .map_err(|err| format!("failed to replace overlay peer {}: {err}", peer.id))?; + if let Err(remove_err) = trust.remove_peer(peer).await { + trust + .remove_dynamic_peer_by_id(&peer.id) + .await + .map_err(|cleanup_err| { + format!( + "failed to replace overlay peer {}: {remove_err}; fallback cleanup \ + failed: {cleanup_err}", + peer.id + ) + })?; + } } } @@ -1373,7 +1383,9 @@ pub(super) async fn revoke_route_overlay( let removed = dynamic_route_overlays.write().await.remove(overlay_id); if let Some(overlay) = removed { for peer in &overlay.peers { - let _ = trust.remove_peer(peer).await; + if trust.remove_peer(peer).await.is_err() { + let _ = trust.remove_dynamic_peer_by_id(&peer.id).await; + } } let mut issuers = dynamic_issuers.write().await; for route_id in overlay.routes.keys() { diff --git a/runtime/router/src/dynamic_caps.rs b/runtime/router/src/dynamic_caps.rs index cd112a0e..e556d04f 100644 --- a/runtime/router/src/dynamic_caps.rs +++ b/runtime/router/src/dynamic_caps.rs @@ -20,19 +20,19 @@ use tokio::time::{Duration, MissedTickBehavior, interval}; use super::*; -const DYNAMIC_CAPS_CONTROL_HELD_LIST_PATH: &str = "/v1/control-state/dynamic-caps/held"; -const DYNAMIC_CAPS_CONTROL_HELD_DETAIL_PATH: &str = "/v1/control-state/dynamic-caps/held/detail"; -const DYNAMIC_CAPS_CONTROL_SHARE_PATH: &str = "/v1/control-state/dynamic-caps/share"; -const DYNAMIC_CAPS_CONTROL_INSPECT_REF_PATH: &str = "/v1/control-state/dynamic-caps/inspect-ref"; -const DYNAMIC_CAPS_CONTROL_REVOKE_PATH: &str = "/v1/control-state/dynamic-caps/revoke"; -const DYNAMIC_CAPS_CONTROL_RESOLVE_ORIGIN_PATH: &str = - "/v1/control-state/dynamic-caps/resolve-origin"; +const DYNAMIC_CAPS_CONTROLLER_HELD_LIST_PATH: &str = "/v1/controller/dynamic-caps/held"; +const DYNAMIC_CAPS_CONTROLLER_HELD_DETAIL_PATH: &str = "/v1/controller/dynamic-caps/held/detail"; +const DYNAMIC_CAPS_CONTROLLER_SHARE_PATH: &str = "/v1/controller/dynamic-caps/share"; +const DYNAMIC_CAPS_CONTROLLER_INSPECT_REF_PATH: &str = "/v1/controller/dynamic-caps/inspect-ref"; +const DYNAMIC_CAPS_CONTROLLER_REVOKE_PATH: &str = "/v1/controller/dynamic-caps/revoke"; +const DYNAMIC_CAPS_CONTROLLER_RESOLVE_ORIGIN_PATH: &str = + "/v1/controller/dynamic-caps/resolve-origin"; const DYNAMIC_CAPS_HANDLE_PREFIX: &str = "/v1/handles/"; const DYNAMIC_CAPS_HANDLE_ID_PREFIX: &str = "hdl_"; const DYNAMIC_CAPS_WATCH_POLL_INTERVAL: Duration = Duration::from_millis(250); #[derive(Debug)] -struct DynamicCapsControlEnv { +struct DynamicCapsControllerEnv { control_url: String, control_auth_token: String, verify_key_raw: String, @@ -184,7 +184,7 @@ impl DynamicCapsRuntime { let Some(listen_addr) = config.dynamic_caps_listen else { return Ok(None); }; - let Some(control_env) = resolve_dynamic_caps_control_env()? else { + let Some(control_env) = resolve_dynamic_caps_controller_env()? else { tracing::warn!( target: "amber.internal", component_id = %config.identity.id, @@ -294,7 +294,7 @@ impl DynamicCapsRuntime { async fn control_held_list(&self) -> Result { self.control_post_json( - DYNAMIC_CAPS_CONTROL_HELD_LIST_PATH, + DYNAMIC_CAPS_CONTROLLER_HELD_LIST_PATH, &ControlDynamicHeldListRequest { holder_component_id: self.component_id.to_string(), }, @@ -307,7 +307,7 @@ impl DynamicCapsRuntime { held_id: &str, ) -> Result { self.control_post_json( - DYNAMIC_CAPS_CONTROL_HELD_DETAIL_PATH, + DYNAMIC_CAPS_CONTROLLER_HELD_DETAIL_PATH, &ControlDynamicHeldDetailRequest { holder_component_id: self.component_id.to_string(), held_id: held_id.to_string(), @@ -321,7 +321,7 @@ impl DynamicCapsRuntime { source: DynamicCapabilityControlSourceRequest, ) -> Result { self.control_post_json( - DYNAMIC_CAPS_CONTROL_RESOLVE_ORIGIN_PATH, + DYNAMIC_CAPS_CONTROLLER_RESOLVE_ORIGIN_PATH, &ControlDynamicResolveOriginRequest { holder_component_id: self.component_id.to_string(), source, @@ -1117,22 +1117,23 @@ fn required_dynamic_caps_env_var( }) } -fn resolve_dynamic_caps_control_env() -> Result, RouterError> { - let control_url = nonempty_env_var(amber_mesh::DYNAMIC_CAPS_CONTROL_URL_ENV); - let control_auth_token = nonempty_env_var(amber_mesh::DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV); +fn resolve_dynamic_caps_controller_env() -> Result, RouterError> { + let control_url = nonempty_env_var(amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV); + let control_auth_token = + nonempty_env_var(amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV); let verify_key_raw = nonempty_env_var(amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV); if control_url.is_none() && control_auth_token.is_none() && verify_key_raw.is_none() { return Ok(None); } let run_id = required_dynamic_caps_env_var(SCENARIO_RUN_ID_ENV, nonempty_env_var(SCENARIO_RUN_ID_ENV))?; - Ok(Some(DynamicCapsControlEnv { + Ok(Some(DynamicCapsControllerEnv { control_url: required_dynamic_caps_env_var( - amber_mesh::DYNAMIC_CAPS_CONTROL_URL_ENV, + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, control_url, )?, control_auth_token: required_dynamic_caps_env_var( - amber_mesh::DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV, + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV, control_auth_token, )?, verify_key_raw: required_dynamic_caps_env_var( @@ -1188,50 +1189,56 @@ mod tests { } #[test] - fn resolve_dynamic_caps_control_env_disables_listener_when_control_env_is_absent() { + fn resolve_dynamic_caps_controller_env_disables_listener_when_control_env_is_absent() { let _guard = ENV_LOCK .get_or_init(|| Mutex::new(())) .lock() .expect("env lock"); let _env = EnvGuard::replace([ - (amber_mesh::DYNAMIC_CAPS_CONTROL_URL_ENV, None), - (amber_mesh::DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV, None), + (amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, None), + ( + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV, + None, + ), (amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, None), (SCENARIO_RUN_ID_ENV, Some("run-1234")), ]); assert!( - resolve_dynamic_caps_control_env() + resolve_dynamic_caps_controller_env() .expect("dynamic caps env should resolve") .is_none(), - "sidecars without dynamic caps control env should leave the listener disabled", + "sidecars without dynamic caps controller env should leave the listener disabled", ); } #[test] - fn resolve_dynamic_caps_control_env_rejects_partial_configuration() { + fn resolve_dynamic_caps_controller_env_rejects_partial_configuration() { let _guard = ENV_LOCK .get_or_init(|| Mutex::new(())) .lock() .expect("env lock"); let _env = EnvGuard::replace([ ( - amber_mesh::DYNAMIC_CAPS_CONTROL_URL_ENV, + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, Some("http://127.0.0.1:24000"), ), - (amber_mesh::DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV, None), + ( + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV, + None, + ), (amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, None), (SCENARIO_RUN_ID_ENV, Some("run-1234")), ]); - let err = resolve_dynamic_caps_control_env().expect_err("partial env must fail"); + let err = resolve_dynamic_caps_controller_env().expect_err("partial env must fail"); assert!( matches!( &err, RouterError::InvalidConfig(message) - if message.contains(amber_mesh::DYNAMIC_CAPS_CONTROL_AUTH_TOKEN_ENV) + if message.contains(amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV) ), - "partial dynamic caps control env should fail with the missing variable name: {err}", + "partial dynamic caps controller env should fail with the missing variable name: {err}", ); } } @@ -1367,7 +1374,7 @@ async fn dynamic_caps_service( }; let response: ShareResponse = match state .control_post_json( - DYNAMIC_CAPS_CONTROL_SHARE_PATH, + DYNAMIC_CAPS_CONTROLLER_SHARE_PATH, &ControlDynamicShareRequest { caller_component_id: state.component_id.to_string(), source, @@ -1425,7 +1432,7 @@ async fn dynamic_caps_service( }; match state .control_post_json::<_, RevokeResponse>( - DYNAMIC_CAPS_CONTROL_REVOKE_PATH, + DYNAMIC_CAPS_CONTROLLER_REVOKE_PATH, &ControlDynamicRevokeRequest { caller_component_id: state.component_id.to_string(), target: source, @@ -1453,7 +1460,7 @@ async fn dynamic_caps_service( } let response: InspectRefResponse = match state .control_post_json( - DYNAMIC_CAPS_CONTROL_INSPECT_REF_PATH, + DYNAMIC_CAPS_CONTROLLER_INSPECT_REF_PATH, &ControlDynamicInspectRefRequest { holder_component_id: state.component_id.to_string(), r#ref: request.r#ref, diff --git a/runtime/router/src/external.rs b/runtime/router/src/external.rs index 122b342e..3707582d 100644 --- a/runtime/router/src/external.rs +++ b/runtime/router/src/external.rs @@ -213,7 +213,7 @@ fn is_disallowed_external_ip(ip: IpAddr) -> bool { } fn allows_loopback_external_target(target: &ExternalTarget) -> bool { - target.url_env == amber_mesh::FRAMEWORK_COMPONENT_CCS_URL_ENV + target.url_env == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV || (target.url_env.starts_with("AMBER_EXTERNAL_SLOT_") && target.url_env.ends_with("_URL")) } diff --git a/runtime/router/src/lib.rs b/runtime/router/src/lib.rs index 82386760..eb12feca 100644 --- a/runtime/router/src/lib.rs +++ b/runtime/router/src/lib.rs @@ -307,7 +307,7 @@ const AMBER_PEER_ID_HEADER: &str = "x-amber-peer-id"; const AMBER_FRAMEWORK_AUTH_HEADER: &str = "x-amber-framework-auth"; fn framework_component_auth_header_value() -> Option { - env::var(amber_mesh::FRAMEWORK_COMPONENT_CCS_AUTH_TOKEN_ENV) + env::var(amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV) .ok() .map(|value| value.trim().to_string()) .filter(|value| !value.is_empty()) diff --git a/runtime/router/src/tests.rs b/runtime/router/src/tests.rs index 7437c2eb..51997ab7 100644 --- a/runtime/router/src/tests.rs +++ b/runtime/router/src/tests.rs @@ -1176,6 +1176,91 @@ async fn revoke_route_overlay_allows_same_peer_id_with_new_key_after_cleanup() { .expect("same peer id should be reusable after overlay revoke cleanup"); } +#[tokio::test] +async fn revoke_route_overlay_cleans_up_stale_dynamic_peer_registration_by_id() { + let config = test_mesh_config(); + let inbound_routes = build_inbound_routes(&config).expect("build inbound routes"); + let trust = TrustBundle::new(&config).expect("trust"); + let dynamic_route_overlays: DynamicRouteOverlays = Arc::new(RwLock::new(HashMap::new())); + let dynamic_issuers: DynamicIssuers = Arc::new(RwLock::new(HashMap::new())); + let original_peer = test_peer("dynamic-peer"); + let stale_peer = test_peer("dynamic-peer"); + let replacement_peer = test_peer("dynamic-peer"); + assert_ne!( + original_peer.public_key, stale_peer.public_key, + "test peers should use distinct keys for the same peer id" + ); + assert_ne!( + stale_peer.public_key, replacement_peer.public_key, + "replacement test peer should also use a distinct key" + ); + + apply_route_overlay( + "overlay-a", + ControlRouteOverlay { + peers: vec![ControlRouteOverlayPeer { + peer_id: original_peer.id.clone(), + peer_key: base64::engine::general_purpose::STANDARD + .encode(original_peer.public_key), + }], + inbound_routes: vec![inbound_route( + "dynamic-route", + "dynamic", + MeshProtocol::Http, + InboundTarget::Local { port: 8080 }, + &["dynamic-peer"], + )], + }, + &trust, + &inbound_routes, + &dynamic_route_overlays, + &dynamic_issuers, + ) + .await + .expect("first overlay should apply"); + + trust + .remove_dynamic_peer_by_id("dynamic-peer") + .await + .expect("simulated drift cleanup should remove the original registration"); + trust + .insert_peer(&stale_peer) + .await + .expect("simulated stale dynamic registration should install"); + + revoke_route_overlay( + "overlay-a", + &trust, + &dynamic_route_overlays, + &dynamic_issuers, + ) + .await; + + apply_route_overlay( + "overlay-b", + ControlRouteOverlay { + peers: vec![ControlRouteOverlayPeer { + peer_id: replacement_peer.id.clone(), + peer_key: base64::engine::general_purpose::STANDARD + .encode(replacement_peer.public_key), + }], + inbound_routes: vec![inbound_route( + "dynamic-route", + "dynamic", + MeshProtocol::Http, + InboundTarget::Local { port: 8080 }, + &["dynamic-peer"], + )], + }, + &trust, + &inbound_routes, + &dynamic_route_overlays, + &dynamic_issuers, + ) + .await + .expect("overlay revoke should clear a stale dynamic peer registration by id"); +} + #[tokio::test] async fn apply_route_overlay_rejects_header_invalid_peer_id() { let config = test_mesh_config(); @@ -1714,7 +1799,7 @@ fn resolve_http_external_target_with_override_rejects_loopback_ip_literals() { fn resolve_http_external_target_with_override_accepts_framework_loopback_ip_literals() { let target = ExternalTarget { name: "component".to_string(), - url_env: amber_mesh::FRAMEWORK_COMPONENT_CCS_URL_ENV.to_string(), + url_env: amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV.to_string(), optional: false, url_override: None, }; diff --git a/runtime/router/src/transport.rs b/runtime/router/src/transport.rs index 21e46095..179b48b5 100644 --- a/runtime/router/src/transport.rs +++ b/runtime/router/src/transport.rs @@ -609,6 +609,21 @@ impl TrustBundle { } Ok(()) } + + pub(super) async fn remove_dynamic_peer_by_id(&self, peer_id: &str) -> Result<(), RouterError> { + let mut inner = self.inner.write().await; + let Some(existing) = inner.noise_by_id.remove(peer_id) else { + return Ok(()); + }; + if existing.static_peer { + inner.noise_by_id.insert(peer_id.to_string(), existing); + return Err(RouterError::Auth(format!( + "peer {peer_id} is statically registered" + ))); + } + inner.id_by_noise.remove(&existing.noise); + Ok(()) + } } fn insert_static_peer( diff --git a/runtime/site-controller/Cargo.toml b/runtime/site-controller/Cargo.toml new file mode 100644 index 00000000..31da23f4 --- /dev/null +++ b/runtime/site-controller/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "amber-site-controller" +edition = "2024" + +[[bin]] +name = "amber-site-controller" +path = "src/bin/amber-site-controller.rs" + +[dependencies] +amber-compiler = { workspace = true } +amber-manifest = { workspace = true } +amber-mesh = { workspace = true } +amber-proxy = { workspace = true } +amber-resolver = { workspace = true } +amber-scenario = { workspace = true } +axum = { version = "0.8.6", features = ["http1", "json", "tokio"] } +base64 = { workspace = true } +miette = { workspace = true, features = ["fancy"] } +reqwest = { workspace = true } +regex = "1.11.1" +rmcp = { workspace = true, features = ["server", "macros", "transport-streamable-http-server", "schemars"] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +serde_yaml = { workspace = true } +tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "signal", "time"] } +url = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/cli/src/framework_component/api.rs b/runtime/site-controller/src/api.rs similarity index 100% rename from cli/src/framework_component/api.rs rename to runtime/site-controller/src/api.rs diff --git a/runtime/site-controller/src/bin/amber-site-controller.rs b/runtime/site-controller/src/bin/amber-site-controller.rs new file mode 100644 index 00000000..c9f588da --- /dev/null +++ b/runtime/site-controller/src/bin/amber-site-controller.rs @@ -0,0 +1,75 @@ +use std::{ + env, + os::unix::process::CommandExt as _, + path::{Path, PathBuf}, + process::Command, +}; + +use miette::{IntoDiagnostic as _, Result, WrapErr as _}; + +fn main() -> Result<()> { + let plan = parse_args()?; + let amber = amber_cli_executable()?; + let err = Command::new(amber) + .arg("run-site-controller") + .arg("--plan") + .arg(plan) + .exec(); + Err(miette::miette!( + "failed to exec amber site controller wrapper: {err}" + )) +} + +fn parse_args() -> Result { + let mut args = env::args_os(); + let _exe = args.next(); + let Some(flag) = args.next() else { + return usage_error(); + }; + if flag != "--plan" { + return usage_error(); + } + let Some(plan) = args.next() else { + return usage_error(); + }; + if args.next().is_some() { + return usage_error(); + } + Ok(PathBuf::from(plan)) +} + +fn usage_error() -> Result { + Err(miette::miette!("usage: amber-site-controller --plan FILE")) +} + +fn amber_cli_executable() -> Result { + if let Some(path) = env::var_os("CARGO_BIN_EXE_amber") { + let path = PathBuf::from(path); + if path.is_file() { + return Ok(path); + } + } + + let current = env::current_exe() + .into_diagnostic() + .wrap_err("failed to resolve amber executable path")?; + if let Some(candidate) = sibling_binary(¤t, "amber") { + return Ok(candidate); + } + Err(miette::miette!( + "failed to locate the amber executable next to {}", + current.display() + )) +} + +fn sibling_binary(current: &Path, name: &str) -> Option { + let exe_name = format!("{name}{}", std::env::consts::EXE_SUFFIX); + let bin_dir = current.parent()?; + for dir in [Some(bin_dir), bin_dir.parent()].into_iter().flatten() { + let candidate = dir.join(&exe_name); + if candidate.is_file() { + return Some(candidate); + } + } + None +} diff --git a/cli/src/framework_component/ccs_api.rs b/runtime/site-controller/src/ccs_api.rs similarity index 73% rename from cli/src/framework_component/ccs_api.rs rename to runtime/site-controller/src/ccs_api.rs index 76be068d..9197cf83 100644 --- a/cli/src/framework_component/ccs_api.rs +++ b/runtime/site-controller/src/ccs_api.rs @@ -1,11 +1,4 @@ -use super::{ - api::*, - http::{forward_create_child, forward_destroy_child}, - orchestration::ProtocolApiError, - planner::CcsApp, - state::FrameworkControlState, - *, -}; +use super::{api::*, state::FrameworkControlState, *}; pub(crate) enum FrameworkComponentInspectRequest { ListTemplates, @@ -90,24 +83,3 @@ pub(crate) async fn execute_framework_component_inspect( } }) } - -pub(crate) async fn execute_framework_component_mutate( - app: &CcsApp, - cap_instance_id: &str, - request: FrameworkComponentMutateRequest, -) -> std::result::Result { - Ok(match request { - FrameworkComponentMutateRequest::CreateChild(request) => { - FrameworkComponentMutateResponse::CreateChild( - forward_create_child(app, cap_instance_id, request).await?, - ) - } - FrameworkComponentMutateRequest::DestroyChild { child } => { - forward_destroy_child(app, cap_instance_id, &child).await?; - FrameworkComponentMutateResponse::DestroyChild(DestroyChildResponse { - child, - destroyed: true, - }) - } - }) -} diff --git a/cli/src/framework_component/control_state_api.rs b/runtime/site-controller/src/control_state_api.rs similarity index 92% rename from cli/src/framework_component/control_state_api.rs rename to runtime/site-controller/src/control_state_api.rs index 3250a4b6..109b4d05 100644 --- a/cli/src/framework_component/control_state_api.rs +++ b/runtime/site-controller/src/control_state_api.rs @@ -1,11 +1,10 @@ use base64::Engine as _; -use serde_json::json; use super::{ dynamic_caps, orchestration::{ - ProtocolApiError, dynamic_capability_component_runtime_endpoint, load_site_actuator_plan, - load_site_manager_state, publish_dynamic_capability_origin, + ProtocolApiError, dynamic_capability_component_runtime_endpoint, load_site_manager_state, + load_site_runtime_plan, publish_dynamic_capability_origin, }, planner::{ControlStateApp, protocol_error}, state::persist_control_state_update, @@ -182,11 +181,11 @@ async fn resolve_dynamic_capability_origin( })?; let holder_runtime = dynamic_capability_component_runtime_endpoint(app, &state, &request.holder_component_id)?; - let holder_plan = load_site_actuator_plan(app, &holder_runtime.site_id)?; + let holder_plan = load_site_runtime_plan(app, &holder_runtime.site_id)?; let origin_runtime = dynamic_capability_component_runtime_endpoint(app, &state, &root.holder_component_id)?; let origin_site_id = origin_runtime.site_id.clone(); - let origin_plan = load_site_actuator_plan(app, &origin_site_id)?; + let origin_plan = load_site_runtime_plan(app, &origin_site_id)?; let origin_manager_state = load_site_manager_state(app, &origin_site_id)?; let origin_peer_id = origin_manager_state .router_identity_id @@ -225,7 +224,8 @@ async fn resolve_dynamic_capability_origin( )) }) .and_then(|router_mesh_addr| { - router_mesh_addr_for_consumer(origin_plan.kind, holder_plan.kind, router_mesh_addr) + app.runtime + .router_mesh_addr_for_consumer(origin_plan.kind, holder_plan.kind, router_mesh_addr) .map_err(|err| { ProtocolApiError::from(protocol_error( ProtocolErrorCode::OriginUnavailable, @@ -236,15 +236,14 @@ async fn resolve_dynamic_capability_origin( )) }) })?; - let overlay_suffix = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode( - serde_json::to_vec(&json!({ - "holder_component_id": request.holder_component_id, - "root_authority_selector": resolved_source.root_authority_selector.clone(), - })) - .expect("dynamic capability origin overlay key should serialize"), + let overlay_id = dynamic_caps::origin_overlay_id( + &request.holder_component_id, + &resolved_source.root_authority_selector, + ); + let route_id = dynamic_caps::origin_route_id( + &request.holder_component_id, + &resolved_source.root_authority_selector, ); - let overlay_id = format!("dynamic-cap-origin-{overlay_suffix}"); - let route_id = format!("dynamic-cap-origin-route-{overlay_suffix}"); let publish = publish_dynamic_capability_origin( app, &origin_site_id, diff --git a/cli/src/framework_component/control_state_mcp.rs b/runtime/site-controller/src/control_state_mcp.rs similarity index 72% rename from cli/src/framework_component/control_state_mcp.rs rename to runtime/site-controller/src/control_state_mcp.rs index 258f44b3..ba21d733 100644 --- a/cli/src/framework_component/control_state_mcp.rs +++ b/runtime/site-controller/src/control_state_mcp.rs @@ -20,21 +20,22 @@ use serde_json::Value; use super::{ control_state_api::{ - self, DynamicCapsInspectRequest, DynamicCapsInspectResponse, DynamicCapsMutateRequest, + DynamicCapsInspectRequest, DynamicCapsInspectResponse, DynamicCapsMutateRequest, DynamicCapsMutateResponse, }, dynamic_caps, - http::authorize_framework_auth_header, - mcp_common::{json_response, map_protocol_api_error}, - planner::ControlStateApp, - *, + mcp_common::{McpOperationResponse, json_response, map_protocol_api_error}, + planner::SiteControllerApp, + site_controller::{ + execute_site_controller_dynamic_caps_inspect, execute_site_controller_dynamic_caps_mutate, + }, }; const HELP_RESOURCE_URI: &str = "amber://framework-dynamic-caps"; const OPERATION_RESOURCE_PREFIX: &str = "amber://framework-dynamic-caps/op/"; pub(crate) fn service( - app: ControlStateApp, + app: SiteControllerApp, ) -> StreamableHttpService { StreamableHttpService::new( move || Ok(FrameworkDynamicCapsMcp::new(app.clone())), @@ -45,12 +46,12 @@ pub(crate) fn service( #[derive(Clone)] pub(crate) struct FrameworkDynamicCapsMcp { - app: ControlStateApp, + app: SiteControllerApp, tool_router: ToolRouter, } impl FrameworkDynamicCapsMcp { - fn new(app: ControlStateApp) -> Self { + fn new(app: SiteControllerApp) -> Self { Self { app, tool_router: Self::tool_router(), @@ -62,66 +63,51 @@ impl FrameworkDynamicCapsMcp { .extensions .get::() .ok_or_else(|| McpError::invalid_request("missing HTTP request context", None))?; - authorize_framework_auth_header(&parts.headers, self.app.control_state_auth_token.as_ref()) - .map_err(map_protocol_api_error) + super::http::authorize_framework_auth_header( + &parts.headers, + self.app.control.control_state_auth_token.as_ref(), + ) + .map_err(map_protocol_api_error) } - fn render_help_resource(&self) -> String { - let mut out = String::from("# framework dynamic caps MCP\n\n"); - out.push_str("Tools:\n"); - out.push_str("- `amber.v1.framework_dynamic_caps.inspect`\n"); - out.push_str("- `amber.v1.framework_dynamic_caps.mutate`\n\n"); - out.push_str("Transport:\n"); - out.push_str("- MCP endpoint: `/mcp`\n"); - out.push_str("- HTTP endpoints remain under `/v1/control-state/dynamic-caps/...`\n\n"); - out.push_str("Inspect operations:\n"); - for op in ["held_list", "held_detail", "inspect_ref", "resolve_origin"] { - out.push_str(&format!( - "- `{op}`: read `{}{op}` for argument details\n", - OPERATION_RESOURCE_PREFIX - )); - } - out.push_str("\nMutate operations:\n"); - for op in ["share", "revoke"] { - out.push_str(&format!( - "- `{op}`: read `{}{op}` for argument details\n", - OPERATION_RESOURCE_PREFIX - )); - } - out + fn help_resource(&self) -> String { + "# framework dynamic caps MCP\n\n- `amber.v1.framework_dynamic_caps.inspect`\n- \ + `amber.v1.framework_dynamic_caps.mutate`\n" + .to_string() } - fn render_operation_resource(&self, name: &str) -> Result { + fn operation_resource(&self, name: &str) -> Result { let doc = match name { "held_list" => { - "# `held_list`\n\nTool: `amber.v1.framework_dynamic_caps.inspect`\n\nArguments:\n\ - ```json\n{ \"op\": \"held_list\", \"holder_component_id\": \"components./alice\" }\n```\n\n\ - Returns `data` matching HTTP `POST /v1/control-state/dynamic-caps/held`." + "# `held_list`\n\nTool: `amber.v1.framework_dynamic_caps.inspect`\n\n```json\n{ \ + \"op\": \"held_list\", \"holder_component_id\": \"components./alice\" }\n```" } "held_detail" => { - "# `held_detail`\n\nTool: `amber.v1.framework_dynamic_caps.inspect`\n\nArguments:\n\ - ```json\n{ \"op\": \"held_detail\", \"holder_component_id\": \"components./alice\", \"held_id\": \"held_root_...\" }\n```\n\n\ - Returns `data` matching HTTP `POST /v1/control-state/dynamic-caps/held/detail`." + "# `held_detail`\n\nTool: `amber.v1.framework_dynamic_caps.inspect`\n\n```json\n{ \ + \"op\": \"held_detail\", \"holder_component_id\": \"components./alice\", \ + \"held_id\": \"held_root_...\" }\n```" } "inspect_ref" => { - "# `inspect_ref`\n\nTool: `amber.v1.framework_dynamic_caps.inspect`\n\nArguments:\n\ - ```json\n{ \"op\": \"inspect_ref\", \"holder_component_id\": \"components./carol\", \"ref\": \"amber://ref/...\" }\n```\n\n\ - Returns `data` matching HTTP `POST /v1/control-state/dynamic-caps/inspect-ref`." + "# `inspect_ref`\n\nTool: `amber.v1.framework_dynamic_caps.inspect`\n\n```json\n{ \ + \"op\": \"inspect_ref\", \"holder_component_id\": \"components./carol\", \"ref\": \ + \"amber://ref/...\" }\n```" } "resolve_origin" => { - "# `resolve_origin`\n\nTool: `amber.v1.framework_dynamic_caps.inspect`\n\nArguments:\n\ - ```json\n{\n \"op\": \"resolve_origin\",\n \"holder_component_id\": \"components./alice\",\n \"source\": {\n \"kind\": \"root_authority\",\n \"root_authority_selector\": {\n \"kind\": \"binding\",\n \"consumer_component_id\": \"components./alice\",\n \"slot_name\": \"upstream\",\n \"provider_component_id\": \"components./provider\",\n \"provider_capability_name\": \"http\"\n }\n }\n}\n```\n\n\ - Returns `data` matching HTTP `POST /v1/control-state/dynamic-caps/resolve-origin`." + "# `resolve_origin`\n\nTool: \ + `amber.v1.framework_dynamic_caps.inspect`\n\n```json\n{\n \"op\": \ + \"resolve_origin\",\n \"holder_component_id\": \"components./alice\",\n \ + \"source\": { \"kind\": \"grant\", \"grant_id\": \"g_...\" }\n}\n```" } "share" => { - "# `share`\n\nTool: `amber.v1.framework_dynamic_caps.mutate`\n\nArguments:\n\ - ```json\n{\n \"op\": \"share\",\n \"caller_component_id\": \"components./alice\",\n \"source\": {\n \"kind\": \"root_authority\",\n \"root_authority_selector\": {\n \"kind\": \"binding\",\n \"consumer_component_id\": \"components./alice\",\n \"slot_name\": \"upstream\",\n \"provider_component_id\": \"components./provider\",\n \"provider_capability_name\": \"http\"\n }\n },\n \"recipient_component_id\": \"components./carol\"\n}\n```\n\n\ - Returns `data` matching HTTP `POST /v1/control-state/dynamic-caps/share`." + "# `share`\n\nTool: `amber.v1.framework_dynamic_caps.mutate`\n\n```json\n{\n \ + \"op\": \"share\",\n \"caller_component_id\": \"components./alice\",\n \ + \"source\": { \"kind\": \"grant\", \"grant_id\": \"g_...\" },\n \ + \"recipient_component_id\": \"components./carol\"\n}\n```" } "revoke" => { - "# `revoke`\n\nTool: `amber.v1.framework_dynamic_caps.mutate`\n\nArguments:\n\ - ```json\n{\n \"op\": \"revoke\",\n \"caller_component_id\": \"components./alice\",\n \"target\": { \"kind\": \"grant\", \"grant_id\": \"g_...\" }\n}\n```\n\n\ - Returns `data` matching HTTP `POST /v1/control-state/dynamic-caps/revoke`." + "# `revoke`\n\nTool: `amber.v1.framework_dynamic_caps.mutate`\n\n```json\n{\n \ + \"op\": \"revoke\",\n \"caller_component_id\": \"components./alice\",\n \ + \"target\": { \"kind\": \"grant\", \"grant_id\": \"g_...\" }\n}\n```" } _ => { return Err(McpError::resource_not_found( @@ -155,7 +141,7 @@ enum RootAuthoritySelectorArg { }, } -impl From for RootAuthoritySelectorIr { +impl From for amber_mesh::dynamic_caps::RootAuthoritySelectorIr { fn from(value: RootAuthoritySelectorArg) -> Self { match value { RootAuthoritySelectorArg::SelfProvide { @@ -264,9 +250,9 @@ impl FrameworkDynamicCapsMcp { &self, Parameters(args): Parameters, context: RequestContext, - ) -> Result, McpError> { + ) -> Result, McpError> { self.authorize(&context)?; - let response = control_state_api::execute_dynamic_caps_inspect( + let response = execute_site_controller_dynamic_caps_inspect( &self.app, match args { InspectArgs::HeldList { @@ -304,10 +290,10 @@ impl FrameworkDynamicCapsMcp { }, ), }, + false, ) .await .map_err(map_protocol_api_error)?; - match response { DynamicCapsInspectResponse::HeldList(data) => json_response("held_list", data), DynamicCapsInspectResponse::HeldDetail(data) => json_response("held_detail", data), @@ -326,9 +312,9 @@ impl FrameworkDynamicCapsMcp { &self, Parameters(args): Parameters, context: RequestContext, - ) -> Result, McpError> { + ) -> Result, McpError> { self.authorize(&context)?; - let response = control_state_api::execute_dynamic_caps_mutate( + let response = execute_site_controller_dynamic_caps_mutate( &self.app, match args { MutateArgs::Share { @@ -352,10 +338,10 @@ impl FrameworkDynamicCapsMcp { target: target.into(), }), }, + false, ) .await .map_err(map_protocol_api_error)?; - match response { DynamicCapsMutateResponse::Share(data) => json_response("share", data), DynamicCapsMutateResponse::Revoke(data) => json_response("revoke", data), @@ -375,9 +361,8 @@ impl ServerHandler for FrameworkDynamicCapsMcp { .with_server_info(Implementation::new("amber-framework-dynamic-caps", "dev")) .with_protocol_version(ProtocolVersion::V_2025_06_18) .with_instructions( - "Use `amber.v1.framework_dynamic_caps.inspect` for reads and \ - `amber.v1.framework_dynamic_caps.mutate` for share or revoke. Read \ - `amber://framework-dynamic-caps` only when you need more detail." + "Use the framework_dynamic_caps tools. Read the amber:// help resources only when \ + needed." .to_string(), ) } @@ -436,16 +421,15 @@ impl ServerHandler for FrameworkDynamicCapsMcp { self.authorize(&context)?; let uri = request.uri; let text = if uri.as_str() == HELP_RESOURCE_URI { - self.render_help_resource() + self.help_resource() } else if let Some(name) = uri.as_str().strip_prefix(OPERATION_RESOURCE_PREFIX) { - self.render_operation_resource(name)? + self.operation_resource(name)? } else { return Err(McpError::resource_not_found( - format!("resource {} not found", uri), + format!("resource {uri} not found"), None, )); }; - Ok(ReadResourceResult::new(vec![ResourceContents::text( text, uri, )])) diff --git a/cli/src/framework_component/dynamic_caps.rs b/runtime/site-controller/src/dynamic_caps.rs similarity index 98% rename from cli/src/framework_component/dynamic_caps.rs rename to runtime/site-controller/src/dynamic_caps.rs index 8cac4f89..59493af4 100644 --- a/cli/src/framework_component/dynamic_caps.rs +++ b/runtime/site-controller/src/dynamic_caps.rs @@ -212,6 +212,39 @@ pub(crate) fn root_authority_key(selector: &RootAuthoritySelectorIr) -> String { .encode(serde_json::to_vec(selector).expect("root authority selector should serialize")) } +fn origin_overlay_suffix( + holder_component_id: &str, + root_authority_selector: &RootAuthoritySelectorIr, +) -> String { + URL_SAFE_NO_PAD.encode( + serde_json::to_vec(&serde_json::json!({ + "holder_component_id": holder_component_id, + "root_authority_selector": root_authority_selector, + })) + .expect("dynamic capability origin overlay key should serialize"), + ) +} + +pub(crate) fn origin_overlay_id( + holder_component_id: &str, + root_authority_selector: &RootAuthoritySelectorIr, +) -> String { + format!( + "dynamic-cap-origin-{}", + origin_overlay_suffix(holder_component_id, root_authority_selector) + ) +} + +pub(crate) fn origin_route_id( + holder_component_id: &str, + root_authority_selector: &RootAuthoritySelectorIr, +) -> String { + format!( + "dynamic-cap-origin-route-{}", + origin_overlay_suffix(holder_component_id, root_authority_selector) + ) +} + pub(crate) fn held_id_for_root(selector: &RootAuthoritySelectorIr) -> String { format!( "{DYNAMIC_CAPABILITY_ROOT_HELD_PREFIX}{}", @@ -257,7 +290,7 @@ pub(crate) fn next_dynamic_grant_id(state: &mut FrameworkControlState) -> String "{DYNAMIC_CAPABILITY_GRANT_ID_PREFIX}{:016x}", state.next_dynamic_capability_grant_id ); - state.next_dynamic_capability_grant_id += 1; + state.next_dynamic_capability_grant_id += state.id_stride.max(1); grant_id } diff --git a/runtime/site-controller/src/http.rs b/runtime/site-controller/src/http.rs new file mode 100644 index 00000000..cb9155e4 --- /dev/null +++ b/runtime/site-controller/src/http.rs @@ -0,0 +1,164 @@ +use std::{ + fs, + io::Write as _, + path::Path, + time::{SystemTime, UNIX_EPOCH}, +}; + +use miette::{IntoDiagnostic as _, Result, WrapErr as _}; +use serde::{Deserialize, Serialize}; + +use super::{ + orchestration::ProtocolApiError, planner::ControlStateApp, state::FRAMEWORK_AUTH_HEADER, *, +}; + +pub(super) async fn healthz() -> Json { + Json(json!({ "ok": true })) +} + +pub(super) async fn cleanup_dynamic_bridge_proxies(app: &ControlStateApp) -> Result<()> { + app.runtime.cleanup().await +} + +pub(super) fn authorize_framework_auth_header( + headers: &HeaderMap, + expected: &str, +) -> std::result::Result<(), ProtocolApiError> { + let actual = required_header(headers, FRAMEWORK_AUTH_HEADER)?; + if actual != expected { + return Err(ProtocolApiError::unauthorized( + "invalid authenticated framework request header", + )); + } + Ok(()) +} + +pub(super) fn required_header( + headers: &HeaderMap, + name: &str, +) -> std::result::Result { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) + .ok_or_else(|| { + ProtocolApiError::unauthorized(format!( + "missing authenticated framework request header `{name}`" + )) + }) +} + +pub(super) async fn shutdown_signal() { + let ctrl_c = async { + signal::ctrl_c() + .await + .expect("framework service should install Ctrl-C handler"); + }; + + #[cfg(unix)] + let terminate = async { + use tokio::signal::unix::{SignalKind, signal}; + + signal(SignalKind::terminate()) + .expect("framework service should install SIGTERM handler") + .recv() + .await; + }; + + #[cfg(unix)] + tokio::select! { + _ = ctrl_c => {} + _ = terminate => {} + } + + #[cfg(not(unix))] + ctrl_c.await; +} + +pub(super) fn write_json(path: &Path, value: &T) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + let bytes = serde_json::to_vec_pretty(value) + .map_err(|err| miette::miette!("failed to serialize {}: {err}", path.display()))?; + write_bytes_atomic(path, &bytes) +} + +pub(super) fn write_bytes_atomic(path: &Path, bytes: &[u8]) -> Result<()> { + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("tmp"); + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + let tmp_path = path.with_file_name(format!(".{file_name}.tmp-{}-{nonce}", std::process::id())); + + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp_path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", tmp_path.display()))?; + if let Err(err) = file.write_all(bytes) { + let _ = fs::remove_file(&tmp_path); + return Err(miette::miette!( + "failed to write {}: {err}", + tmp_path.display() + )); + } + if let Err(err) = file.sync_all() { + let _ = fs::remove_file(&tmp_path); + return Err(miette::miette!( + "failed to sync {}: {err}", + tmp_path.display() + )); + } + drop(file); + + fs::rename(&tmp_path, path) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to replace {} with {}", + path.display(), + tmp_path.display() + ) + })?; + sync_parent_directory(path)?; + Ok(()) +} + +pub(super) fn sync_parent_directory(path: &Path) -> Result<()> { + #[cfg(unix)] + { + if let Some(parent) = path.parent() { + fs::File::open(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to open parent directory {}", parent.display()))? + .sync_all() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to sync parent directory {}", parent.display()) + })?; + } + } + #[cfg(not(unix))] + { + let _ = path; + } + Ok(()) +} + +pub(super) fn read_json Deserialize<'de>>(path: &Path, label: &str) -> Result { + let bytes = fs::read(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {label} {}", path.display()))?; + serde_json::from_slice(&bytes) + .map_err(|err| miette::miette!("invalid {label} {}: {err}", path.display())) +} diff --git a/cli/src/framework_component/mod.rs b/runtime/site-controller/src/lib.rs similarity index 63% rename from cli/src/framework_component/mod.rs rename to runtime/site-controller/src/lib.rs index 2da16e1c..123b6454 100644 --- a/cli/src/framework_component/mod.rs +++ b/runtime/site-controller/src/lib.rs @@ -1,13 +1,11 @@ use std::{ collections::{BTreeMap, BTreeSet}, - fs, future::Future, - io::Write as _, - net::{Ipv4Addr, SocketAddr}, + net::SocketAddr, path::{Path, PathBuf}, pin::Pin, sync::Arc, - time::{Duration, SystemTime, UNIX_EPOCH}, + time::Duration, }; use amber_compiler::{ @@ -41,7 +39,7 @@ use amber_mesh::{ use amber_proxy::{ apply_route_overlay_with_retry, clear_external_slot_with_retry, register_export_peer_with_retry, register_external_slot_with_retry, - unregister_export_peer_with_retry, + revoke_route_overlay_with_retry, unregister_export_peer_with_retry, }; use amber_resolver::{Backend, RemoteResolver, Resolution, Resolver}; use amber_scenario::{ @@ -50,30 +48,16 @@ use amber_scenario::{ ir::{BindingFromIr, BindingIr, ComponentExportTargetIr, ComponentIr, ManifestCatalogEntryIr}, }; use axum::{ - Json, Router, - extract::{Path as AxumPath, State}, + Json, http::{HeaderMap, StatusCode}, response::{IntoResponse, Response}, - routing::{get, post}, }; use base64::Engine as _; use miette::{IntoDiagnostic as _, Result, WrapErr as _}; use reqwest::Client as ReqwestClient; use serde::{Deserialize, Serialize}; use serde_json::json; -use tokio::{net::TcpListener, signal, sync::Mutex}; - -use crate::mixed_run::{ - BridgeProxyHandle, BridgeProxyKey, DesiredExportPeerOverlay, DesiredExternalSlotOverlay, - LaunchedSite, LiveComponentRuntimeMetadata, SiteActuatorPlan, SiteReceipt, - clear_desired_overlay_for_consumer, clear_desired_overlay_for_provider, - collect_live_component_runtime_metadata, host_service_bind_addr_for_consumer, - host_service_host_for_consumer, launched_site_from_receipt, load_live_site_router_mesh_config, - parse_control_endpoint, project_kubernetes_dynamic_child_artifact_files, - read_json as read_run_json, resolve_link_external_url_for_output, - router_mesh_addr_for_consumer, site_actuator_child_root_for_site, site_state_path, - stop_bridge_proxies, update_desired_overlay_for_consumer, update_desired_overlay_for_provider, -}; +use tokio::{signal, sync::Mutex}; mod api; mod ccs_api; @@ -85,19 +69,27 @@ mod mcp; mod mcp_common; mod orchestration; mod planner; +mod runtime_api; +mod site_controller; mod state; #[cfg(test)] mod tests; -pub(crate) use self::{ - http::{run_framework_ccs, run_framework_control_state}, - orchestration::{ - SiteActuatorDestroyRequest, SiteActuatorPrepareRequest, SiteActuatorPublishRequest, +pub use self::{ + runtime_api::{ + DesiredExportPeerOverlay, DesiredExternalSlotOverlay, LaunchedSite, + LiveComponentRuntimeMetadata, SharedSiteControllerRuntime, SiteControllerRuntime, + SiteControllerRuntimeFuture, SiteControllerRuntimePlan, SiteReceipt, + launched_site_from_receipt, parse_control_endpoint, + project_kubernetes_dynamic_child_artifact_files, site_controller_plan_path, + site_controller_runtime_child_root_for_site, + site_controller_runtime_plan_from_controller_plan, site_state_path, }, + site_controller::run_site_controller, state::{ DynamicInputRouteRecord, DynamicInputRouteTarget, DynamicProxyExportRecord, - DynamicSitePlanRecord, build_control_state, ccs_listen_addr_for_site, ccs_url_for_site, - control_state_service_url, generate_framework_auth_token, write_control_state, - write_control_state_service_plan, write_framework_ccs_plan, + DynamicSitePlanRecord, SiteControllerPeerPlan, SiteControllerPlan, + authority_url_for_listen_addr, build_site_controller_state, generate_framework_auth_token, + write_control_state, write_site_controller_plan, }, }; diff --git a/cli/src/framework_component/mcp.rs b/runtime/site-controller/src/mcp.rs similarity index 91% rename from cli/src/framework_component/mcp.rs rename to runtime/site-controller/src/mcp.rs index cb0ae3d1..0e7571b1 100644 --- a/cli/src/framework_component/mcp.rs +++ b/runtime/site-controller/src/mcp.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use amber_manifest::ManifestRef; +use amber_mesh::component_protocol::{CreateChildRequest, TemplateResolveRequest}; use axum::http::request::Parts; use rmcp::{ ErrorData as McpError, Json, RoleServer, ServerHandler, @@ -23,21 +24,22 @@ use serde_json::Value; use super::{ ccs_api::{ - self, FrameworkComponentInspectRequest, FrameworkComponentInspectResponse, + FrameworkComponentInspectRequest, FrameworkComponentInspectResponse, FrameworkComponentMutateRequest, FrameworkComponentMutateResponse, }, - http::*, - mcp_common::{McpOperationResponse, json_response, map_protocol_api_error, map_protocol_error}, - planner::CcsApp, - state::{CapabilityInstanceRecord, FrameworkControlState}, - *, + mcp_common::{McpOperationResponse, json_response, map_protocol_api_error}, + planner::SiteControllerApp, + site_controller::{ + authorize_public_request, execute_site_controller_framework_inspect, + execute_site_controller_framework_mutate, + }, }; const HELP_RESOURCE_URI: &str = "amber://framework-component"; const OPERATION_RESOURCE_PREFIX: &str = "amber://framework-component/op/"; pub(crate) fn service( - app: CcsApp, + app: SiteControllerApp, ) -> StreamableHttpService { StreamableHttpService::new( move || Ok(FrameworkComponentMcp::new(app.clone())), @@ -48,12 +50,12 @@ pub(crate) fn service( #[derive(Clone)] pub(crate) struct FrameworkComponentMcp { - app: CcsApp, + app: SiteControllerApp, tool_router: ToolRouter, } impl FrameworkComponentMcp { - fn new(app: CcsApp) -> Self { + fn new(app: SiteControllerApp) -> Self { Self { app, tool_router: Self::tool_router(), @@ -63,17 +65,23 @@ impl FrameworkComponentMcp { async fn authorize( &self, context: &RequestContext, - ) -> Result<(CapabilityInstanceRecord, FrameworkControlState), McpError> { + ) -> Result< + ( + super::state::CapabilityInstanceRecord, + super::state::FrameworkControlState, + ), + McpError, + > { let parts = context .extensions .get::() .ok_or_else(|| McpError::invalid_request("missing HTTP request context", None))?; - authorize_request(&self.app, &parts.headers) + authorize_public_request(&self.app, &parts.headers) .await .map_err(map_protocol_api_error) } - fn render_help_resource(&self) -> String { + fn help_resource(&self) -> String { let mut out = String::from("# framework.component MCP\n\n"); out.push_str("Tools:\n"); out.push_str("- `amber.v1.framework_component.inspect`\n"); @@ -106,7 +114,7 @@ impl FrameworkComponentMcp { out } - fn render_operation_resource(&self, name: &str) -> Result { + fn operation_resource(&self, name: &str) -> Result { let doc = match name { "list_templates" => { "# `list_templates`\n\nTool: `amber.v1.framework_component.inspect`\n\nArguments:\n\ @@ -259,9 +267,10 @@ impl FrameworkComponentMcp { context: RequestContext, ) -> Result, McpError> { let (record, state) = self.authorize(&context).await?; - let response = ccs_api::execute_framework_component_inspect( + let response = execute_site_controller_framework_inspect( + &self.app, + &record, &state, - record.authority_realm_id, match args { InspectArgs::ListTemplates => FrameworkComponentInspectRequest::ListTemplates, InspectArgs::GetTemplate { template } => { @@ -281,7 +290,7 @@ impl FrameworkComponentMcp { }, ) .await - .map_err(map_protocol_error)?; + .map_err(map_protocol_api_error)?; match response { FrameworkComponentInspectResponse::ListTemplates(data) => { json_response("list_templates", data) @@ -311,10 +320,11 @@ impl FrameworkComponentMcp { Parameters(args): Parameters, context: RequestContext, ) -> Result, McpError> { - let (record, _) = self.authorize(&context).await?; - let response = ccs_api::execute_framework_component_mutate( + let (record, state) = self.authorize(&context).await?; + let response = execute_site_controller_framework_mutate( &self.app, - &record.cap_instance_id, + &record, + &state, match args { MutateArgs::CreateChild { template, @@ -374,7 +384,7 @@ impl ServerHandler for FrameworkComponentMcp { request: InitializeRequestParams, context: RequestContext, ) -> Result { - self.authorize(&context).await?; + let _ = self.authorize(&context).await?; if context.peer.peer_info().is_none() { context.peer.set_peer_info(request); } @@ -386,7 +396,7 @@ impl ServerHandler for FrameworkComponentMcp { _request: Option, context: RequestContext, ) -> Result { - self.authorize(&context).await?; + let _ = self.authorize(&context).await?; Ok(ListResourcesResult { resources: vec![ RawResource::new(HELP_RESOURCE_URI, "framework.component MCP").no_annotation(), @@ -401,7 +411,7 @@ impl ServerHandler for FrameworkComponentMcp { _request: Option, context: RequestContext, ) -> Result { - self.authorize(&context).await?; + let _ = self.authorize(&context).await?; Ok(ListResourceTemplatesResult { resource_templates: vec![ RawResourceTemplate::new( @@ -420,19 +430,18 @@ impl ServerHandler for FrameworkComponentMcp { request: ReadResourceRequestParams, context: RequestContext, ) -> Result { - self.authorize(&context).await?; + let _ = self.authorize(&context).await?; let uri = request.uri; let text = if uri.as_str() == HELP_RESOURCE_URI { - self.render_help_resource() + self.help_resource() } else if let Some(name) = uri.as_str().strip_prefix(OPERATION_RESOURCE_PREFIX) { - self.render_operation_resource(name)? + self.operation_resource(name)? } else { return Err(McpError::resource_not_found( - format!("resource {} not found", uri), + format!("resource {uri} not found"), None, )); }; - Ok(ReadResourceResult::new(vec![ResourceContents::text( text, uri, )])) diff --git a/cli/src/framework_component/mcp_common.rs b/runtime/site-controller/src/mcp_common.rs similarity index 100% rename from cli/src/framework_component/mcp_common.rs rename to runtime/site-controller/src/mcp_common.rs diff --git a/cli/src/framework_component/orchestration.rs b/runtime/site-controller/src/orchestration.rs similarity index 77% rename from cli/src/framework_component/orchestration.rs rename to runtime/site-controller/src/orchestration.rs index ad5f3ae9..3a5ffde1 100644 --- a/cli/src/framework_component/orchestration.rs +++ b/runtime/site-controller/src/orchestration.rs @@ -1,3 +1,6 @@ +#[cfg(target_os = "linux")] +use std::net::Ipv4Addr; + use amber_mesh::{ InboundRoute, InboundTarget, MeshConfigPublic, MeshPeer, OutboundRoute, router_external_route_id, @@ -6,17 +9,17 @@ use amber_mesh::{ use super::{http::*, planner::*, state::*, *}; #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct SiteActuatorPrepareRequest { +pub(crate) struct SiteControllerPrepareRequest { pub(crate) site_plan: DynamicSitePlanRecord, } #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct SiteActuatorPublishRequest { +pub(crate) struct SiteControllerPublishRequest { pub(crate) site_plan: DynamicSitePlanRecord, } #[derive(Clone, Debug, Serialize, Deserialize, Default)] -pub(crate) struct SiteActuatorDestroyRequest { +pub(crate) struct SiteControllerDestroyRequest { #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) desired_site_plan: Option, } @@ -121,7 +124,7 @@ pub(super) fn control_state_step_error( ) } -pub(super) fn actuator_protocol_error( +pub(super) fn controller_protocol_error( code: ProtocolErrorCode, site_id: &str, action: &str, @@ -137,16 +140,8 @@ pub(super) fn site_state_root_for(app: &ControlStateApp, site_id: &str) -> PathB Path::new(&app.state_root).join(site_id) } -pub(super) fn site_actuator_plan_path_for_site(app: &ControlStateApp, site_id: &str) -> PathBuf { - site_state_root_for(app, site_id).join("site-actuator-plan.json") -} - -pub(super) fn framework_ccs_plan_path_for_site(app: &ControlStateApp, site_id: &str) -> PathBuf { - site_state_root_for(app, site_id).join("framework-ccs-plan.json") -} - -pub(super) fn site_actuator_base_url(plan: &SiteActuatorPlan) -> String { - format!("http://{}", plan.listen_addr) +pub(super) fn site_controller_plan_path_for_site(app: &ControlStateApp, site_id: &str) -> PathBuf { + site_controller_plan_path(&site_state_root_for(app, site_id)) } pub(super) fn site_receipt_from_manager_state(state: &SiteManagerStateView) -> SiteReceipt { @@ -163,6 +158,8 @@ pub(super) fn site_receipt_from_manager_state(state: &SiteManagerStateView) -> S router_mesh_addr: state.router_mesh_addr.clone(), router_identity_id: state.router_identity_id.clone(), router_public_key_b64: state.router_public_key_b64.clone(), + site_controller_pid: state.site_controller_pid, + site_controller_url: state.site_controller_url.clone(), } } @@ -215,7 +212,7 @@ pub(super) fn load_site_manager_state( app: &ControlStateApp, site_id: &str, ) -> std::result::Result { - read_run_json( + read_json( &site_state_path(&app.state_root, site_id), "site manager state", ) @@ -247,58 +244,72 @@ pub(super) fn load_launched_site( }) } -pub(super) fn load_site_actuator_plan( +pub(super) fn load_site_controller_plan( app: &ControlStateApp, site_id: &str, -) -> std::result::Result { - let path = site_actuator_plan_path_for_site(app, site_id); - read_json(&path, "site actuator plan").map_err(|err| { +) -> std::result::Result { + if site_id == app.controller_plan.site_id { + return Ok(app.controller_plan.as_ref().clone()); + } + let path = site_controller_plan_path_for_site(app, site_id); + read_json(&path, "site controller plan").map_err(|err| { protocol_error( ProtocolErrorCode::SiteNotActive, - &format!("site `{site_id}` actuator plan is unavailable: {err}"), + &format!("site `{site_id}` controller plan is unavailable: {err}"), ) }) } -pub(super) fn load_framework_ccs_plan( +pub(super) fn load_site_runtime_plan( app: &ControlStateApp, site_id: &str, -) -> std::result::Result { - let path = framework_ccs_plan_path_for_site(app, site_id); - read_json(&path, "framework CCS plan").map_err(|err| { - protocol_error( - ProtocolErrorCode::SiteNotActive, - &format!("site `{site_id}` framework CCS plan is unavailable: {err}"), - ) - }) +) -> std::result::Result { + Ok(site_controller_runtime_plan_from_controller_plan( + &load_site_controller_plan(app, site_id)?, + )) } -pub(super) fn framework_ccs_base_url(plan: &FrameworkCcsPlan) -> String { - format!("http://{}", plan.listen_addr) +fn controller_authority_url<'a>( + app: &'a ControlStateApp, + site_id: &str, +) -> std::result::Result<&'a str, ProtocolErrorResponse> { + if site_id == app.controller_plan.site_id { + return Ok(app.controller_plan.authority_url.as_str()); + } + app.peer_controllers + .get(site_id) + .map(|controller| controller.authority_url.as_str()) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::SiteNotActive, + &format!("site `{site_id}` controller metadata is unavailable"), + ) + }) } -pub(super) fn load_site_actuator_plan_at( +pub(super) fn load_site_runtime_plan_at( site_state_root: &Path, -) -> std::result::Result { - read_json( - &site_state_root.join("site-actuator-plan.json"), - "site actuator plan", +) -> std::result::Result { + let plan: SiteControllerPlan = read_json( + &site_controller_plan_path(site_state_root), + "site controller plan", ) .map_err(|err| { protocol_error( ProtocolErrorCode::SiteNotActive, &format!( - "site actuator plan under `{}` is unavailable: {err}", + "site controller plan under `{}` is unavailable: {err}", site_state_root.display() ), ) - }) + })?; + Ok(site_controller_runtime_plan_from_controller_plan(&plan)) } pub(super) fn load_site_manager_state_at( site_state_root: &Path, ) -> std::result::Result { - read_run_json( + read_json( &site_state_root.join("manager-state.json"), "site manager state", ) @@ -319,10 +330,9 @@ pub(super) async fn publish_dynamic_capability_origin( request: &dynamic_caps::PublishDynamicCapabilityOriginRequest, ) -> std::result::Result { - let plan = load_framework_ccs_plan(app, site_id)?; let url = format!( "{}/v1/internal/dynamic-caps/origins/publish", - framework_ccs_base_url(&plan).trim_end_matches('/') + controller_authority_url(app, site_id)?.trim_end_matches('/') ); let response = app .client @@ -335,7 +345,7 @@ pub(super) async fn publish_dynamic_capability_origin( protocol_error( ProtocolErrorCode::OriginUnavailable, &format!( - "failed to reach framework CCS on site `{site_id}` while publishing dynamic \ + "failed to reach site controller on site `{site_id}` while publishing dynamic \ capability origin: {err}" ), ) @@ -345,7 +355,7 @@ pub(super) async fn publish_dynamic_capability_origin( protocol_error( ProtocolErrorCode::OriginUnavailable, &format!( - "framework CCS on site `{site_id}` returned invalid JSON while publishing \ + "site controller on site `{site_id}` returned invalid JSON while publishing \ dynamic capability origin: {err}" ), ) @@ -356,8 +366,8 @@ pub(super) async fn publish_dynamic_capability_origin( protocol_error( ProtocolErrorCode::OriginUnavailable, &format!( - "failed to read framework CCS error response on site `{site_id}` while publishing \ - dynamic capability origin: {err}" + "failed to read site controller error response on site `{site_id}` while \ + publishing dynamic capability origin: {err}" ), ) })?; @@ -367,7 +377,7 @@ pub(super) async fn publish_dynamic_capability_origin( Err(protocol_error( ProtocolErrorCode::OriginUnavailable, &format!( - "framework CCS on site `{site_id}` returned {status} while publishing dynamic \ + "site controller on site `{site_id}` returned {status} while publishing dynamic \ capability origin" ), )) @@ -392,8 +402,10 @@ pub(super) fn dynamic_capability_component_runtime_endpoint( &format!("live component `{logical_component_id}` is not assigned to a live site"), ) })?; - let site_plan = load_site_actuator_plan(app, site_id)?; - let runtime = collect_live_component_runtime_metadata(&site_plan) + let site_plan = load_site_runtime_plan(app, site_id)?; + let runtime = app + .runtime + .collect_live_component_runtime_metadata(&site_plan) .map_err(|err| { protocol_error( ProtocolErrorCode::OriginUnavailable, @@ -654,10 +666,10 @@ pub(super) fn dynamic_capability_allowed_mesh_peers( } pub(super) async fn publish_dynamic_capability_origin_local( - app: &CcsApp, + app: &LocalDynamicCapabilityOriginApp, request: dynamic_caps::PublishDynamicCapabilityOriginRequest, ) -> std::result::Result { - let site_plan = load_site_actuator_plan_at(&app.site_state_root)?; + let site_plan = load_site_runtime_plan_at(&app.site_state_root)?; let holder_component_id = match &request.root_authority_selector { RootAuthoritySelectorIr::SelfProvide { component_id, .. } => component_id.clone(), RootAuthoritySelectorIr::Binding { @@ -670,8 +682,10 @@ pub(super) async fn publish_dynamic_capability_origin_local( } => consumer_component_id.clone(), }; let holder_moniker = dynamic_caps::moniker_from_logical_component_id(&holder_component_id)?; - let mut site_components = - collect_live_component_runtime_metadata(&site_plan).map_err(|err| { + let mut site_components = app + .runtime + .collect_live_component_runtime_metadata(&site_plan) + .map_err(|err| { protocol_error( ProtocolErrorCode::OriginUnavailable, &format!( @@ -680,15 +694,18 @@ pub(super) async fn publish_dynamic_capability_origin_local( ), ) })?; - let site_router = load_live_site_router_mesh_config(&site_plan).map_err(|err| { - protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "failed to collect live router metadata on site `{}`: {err}", - site_plan.site_id - ), - ) - })?; + let site_router = app + .runtime + .load_live_site_router_mesh_config(&site_plan) + .map_err(|err| { + protocol_error( + ProtocolErrorCode::OriginUnavailable, + &format!( + "failed to collect live router metadata on site `{}`: {err}", + site_plan.site_id + ), + ) + })?; let runtime = site_components.remove(holder_moniker).ok_or_else(|| { protocol_error( ProtocolErrorCode::OriginUnavailable, @@ -767,7 +784,7 @@ pub(super) async fn publish_dynamic_capability_origin_local( }) } -pub(super) async fn call_site_actuator( +pub(super) async fn call_site_controller( app: &ControlStateApp, site_id: &str, path: &str, @@ -775,28 +792,105 @@ pub(super) async fn call_site_actuator( error_code: ProtocolErrorCode, action: &str, ) -> std::result::Result<(), ProtocolErrorResponse> { - let plan = load_site_actuator_plan(app, site_id)?; - let url = format!("{}{}", site_actuator_base_url(&plan), path); - let request = app.client.post(url); - let request = if let Some(body) = body { - request.json(body) + if site_id == app.controller_plan.site_id { + let local_child_id = || -> std::result::Result { + path.rsplit('/') + .nth(1) + .ok_or_else(|| { + controller_protocol_error( + error_code, + site_id, + action, + format!("site controller path `{path}` is malformed"), + ) + })? + .parse() + .map_err(|err| { + controller_protocol_error( + error_code, + site_id, + action, + format!("site controller child id in `{path}` is malformed: {err}"), + ) + }) + }; + match path { + path if path.ends_with("/prepare") => { + let request = serde_json::to_value(body).expect("site op body should serialize"); + let request = serde_json::from_value::(request) + .expect("prepare request should deserialize"); + return app + .runtime + .prepare_child(&app.controller_plan, local_child_id()?, request.site_plan) + .await + .map_err(|err| controller_protocol_error(error_code, site_id, action, err)); + } + path if path.ends_with("/publish") => { + let request = serde_json::to_value(body).expect("site op body should serialize"); + let request = serde_json::from_value::(request) + .expect("publish request should deserialize"); + return app + .runtime + .publish_child(&app.controller_plan, local_child_id()?, request.site_plan) + .await + .map_err(|err| controller_protocol_error(error_code, site_id, action, err)); + } + path if path.ends_with("/destroy") => { + let request = serde_json::to_value(body).expect("site op body should serialize"); + let request = serde_json::from_value::(request) + .expect("destroy request should deserialize"); + return app + .runtime + .destroy_child( + &app.controller_plan, + local_child_id()?, + request.desired_site_plan, + ) + .await + .map_err(|err| controller_protocol_error(error_code, site_id, action, err)); + } + _ => {} + } } else { - request - }; - let response = request - .send() - .await - .map_err(|err| actuator_protocol_error(error_code, site_id, action, err))?; - if response.status().is_success() { - return Ok(()); + let controller = app.peer_controllers.get(site_id).ok_or_else(|| { + controller_protocol_error( + error_code, + site_id, + action, + "site controller is not present in peer metadata", + ) + })?; + let url = format!("{}{}", controller.authority_url.trim_end_matches('/'), path); + let request = app + .client + .post(url) + .header(FRAMEWORK_AUTH_HEADER, app.control_state_auth_token.as_ref()); + let request = if let Some(body) = body { + request.json(body) + } else { + request + }; + let response = request + .send() + .await + .map_err(|err| controller_protocol_error(error_code, site_id, action, err))?; + if response.status().is_success() { + return Ok(()); + } + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(controller_protocol_error( + error_code, + site_id, + action, + format!("HTTP {status}: {}", body.trim()), + )); } - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - Err(actuator_protocol_error( + Err(controller_protocol_error( error_code, site_id, action, - format!("HTTP {status}: {}", body.trim()), + format!("unsupported site controller action path `{path}`"), )) } @@ -805,12 +899,12 @@ pub(super) async fn prepare_child_on_site( child_id: u64, site_plan: &DynamicSitePlanRecord, ) -> std::result::Result<(), ProtocolErrorResponse> { - let path = format!("/v1/children/{child_id}/prepare"); - call_site_actuator( + let path = format!("/v1/controller/site/children/{child_id}/prepare"); + call_site_controller( app, &site_plan.site_id, &path, - Some(&SiteActuatorPrepareRequest { + Some(&SiteControllerPrepareRequest { site_plan: site_plan.clone(), }), ProtocolErrorCode::PrepareFailed, @@ -824,12 +918,12 @@ pub(super) async fn publish_child_on_site( child_id: u64, site_plan: &DynamicSitePlanRecord, ) -> std::result::Result<(), ProtocolErrorResponse> { - let path = format!("/v1/children/{child_id}/publish"); - call_site_actuator( + let path = format!("/v1/controller/site/children/{child_id}/publish"); + call_site_controller( app, &site_plan.site_id, &path, - Some(&SiteActuatorPublishRequest { + Some(&SiteControllerPublishRequest { site_plan: site_plan.clone(), }), ProtocolErrorCode::PublishFailed, @@ -838,11 +932,11 @@ pub(super) async fn publish_child_on_site( .await } -fn site_actuator_child_needs_prepare(err: &ProtocolErrorResponse, child_id: u64) -> bool { +fn site_controller_child_needs_prepare(err: &ProtocolErrorResponse, child_id: u64) -> bool { err.code == ProtocolErrorCode::PublishFailed && err .message - .contains(&format!("site actuator child {child_id} is not prepared")) + .contains(&format!("site controller child {child_id} is not prepared")) } pub(super) async fn publish_child_on_site_with_prepare_retry( @@ -852,7 +946,7 @@ pub(super) async fn publish_child_on_site_with_prepare_retry( ) -> std::result::Result<(), ProtocolErrorResponse> { match publish_child_on_site(app, child_id, site_plan).await { Ok(()) => Ok(()), - Err(err) if site_actuator_child_needs_prepare(&err, child_id) => { + Err(err) if site_controller_child_needs_prepare(&err, child_id) => { prepare_child_on_site(app, child_id, site_plan).await?; publish_child_on_site(app, child_id, site_plan).await } @@ -865,13 +959,25 @@ pub(super) async fn rollback_child_on_site( child_id: u64, site_id: &str, ) -> Result<()> { - let path = format!("/v1/children/{child_id}/rollback"); - let plan = load_site_actuator_plan(app, site_id) - .map_err(|err| miette::miette!("failed to load site actuator plan: {}", err.message))?; - let url = format!("{}{}", site_actuator_base_url(&plan), path); + if site_id == app.controller_plan.site_id { + return app + .runtime + .rollback_child(&app.controller_plan, child_id) + .await; + } + let path = format!("/v1/controller/site/children/{child_id}/rollback"); + let controller = app + .peer_controllers + .get(site_id) + .ok_or_else(|| miette::miette!("site `{site_id}` controller metadata is unavailable"))?; let response = app .client - .post(url) + .post(format!( + "{}{}", + controller.authority_url.trim_end_matches('/'), + path + )) + .header(FRAMEWORK_AUTH_HEADER, app.control_state_auth_token.as_ref()) .send() .await .into_diagnostic() @@ -893,12 +999,12 @@ pub(super) async fn destroy_child_on_site( site_id: &str, desired_site_plan: Option, ) -> std::result::Result<(), ProtocolErrorResponse> { - let path = format!("/v1/children/{child_id}/destroy"); - call_site_actuator( + let path = format!("/v1/controller/site/children/{child_id}/destroy"); + call_site_controller( app, site_id, &path, - Some(&SiteActuatorDestroyRequest { desired_site_plan }), + Some(&SiteControllerDestroyRequest { desired_site_plan }), ProtocolErrorCode::ControlStateUnavailable, "destroy child", ) @@ -917,26 +1023,24 @@ pub(super) async fn publish_external_slot_overlay( let consumer = load_launched_site(app, &link.consumer_site)?; let provider_output_dir = provider_output_dir_for_link(app, child, Path::new(&provider.receipt.artifact_dir), link); - let external_url = { - let mut bridge_proxies = app.bridge_proxies.lock().await; - resolve_link_external_url_for_output( + let external_url = app + .runtime + .resolve_link_external_url( &provider, &provider_output_dir, link, consumer.receipt.kind, &app.run_root, - &mut bridge_proxies, ) .await .map_err(|err| { - actuator_protocol_error( + controller_protocol_error( ProtocolErrorCode::PublishFailed, &link.consumer_site, "compute external slot overlay", err, ) - })? - }; + })?; register_external_slot_with_retry( &consumer.router_control, &link.external_slot_name, @@ -945,29 +1049,30 @@ pub(super) async fn publish_external_slot_overlay( ) .await .map_err(|err| { - actuator_protocol_error( + controller_protocol_error( ProtocolErrorCode::PublishFailed, &link.consumer_site, "publish external slot overlay", err, ) })?; - update_desired_overlay_for_consumer( - &site_state_root_for(app, &link.consumer_site), - overlay_id, - DesiredExternalSlotOverlay { - slot_name: link.external_slot_name.clone(), - url: external_url, - }, - ) - .map_err(|err| { - actuator_protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &link.consumer_site, - "persist desired external slot overlay", - err, + app.runtime + .update_desired_overlay_for_consumer( + &site_state_root_for(app, &link.consumer_site), + overlay_id, + DesiredExternalSlotOverlay { + slot_name: link.external_slot_name.clone(), + url: external_url, + }, ) - }) + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &link.consumer_site, + "persist desired external slot overlay", + err, + ) + }) } pub(super) async fn publish_export_peer_overlay( @@ -994,32 +1099,33 @@ pub(super) async fn publish_export_peer_overlay( ) .await .map_err(|err| { - actuator_protocol_error( + controller_protocol_error( ProtocolErrorCode::PublishFailed, &link.provider_site, "publish export-peer overlay", err, ) })?; - update_desired_overlay_for_provider( - &site_state_root_for(app, &link.provider_site), - overlay_id, - DesiredExportPeerOverlay { - export_name: link.export_name.clone(), - peer_id: consumer.router_identity.id, - peer_key_b64: consumer_key, - protocol: link.protocol.to_string(), - route_id: Some(route_id), - }, - ) - .map_err(|err| { - actuator_protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &link.provider_site, - "persist desired export-peer overlay", - err, + app.runtime + .update_desired_overlay_for_provider( + &site_state_root_for(app, &link.provider_site), + overlay_id, + DesiredExportPeerOverlay { + export_name: link.export_name.clone(), + peer_id: consumer.router_identity.id, + peer_key_b64: consumer_key, + protocol: link.protocol.to_string(), + route_id: Some(route_id), + }, ) - }) + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &link.provider_site, + "persist desired export-peer overlay", + err, + ) + }) } pub(super) fn child_link_records(child: &LiveChildRecord) -> Vec { @@ -1081,7 +1187,7 @@ pub(super) fn provider_output_dir_for_link( if !provider_in_child { return provider_artifact_dir.to_path_buf(); } - site_actuator_child_root_for_site( + site_controller_runtime_child_root_for_site( &site_state_root_for(app, &link.provider_site), child.child_id, ) @@ -1176,9 +1282,13 @@ pub(super) async fn clear_external_slot_overlay( let overlay_id = overlay_id_for_link_action(child, link, |action| { matches!(action, DynamicOverlayAction::ExternalSlot { .. }) })?; - clear_desired_overlay_for_consumer(&site_state_root_for(app, &link.consumer_site), overlay_id) + app.runtime + .clear_desired_overlay_for_consumer( + &site_state_root_for(app, &link.consumer_site), + overlay_id, + ) .map_err(|err| { - actuator_protocol_error( + controller_protocol_error( ProtocolErrorCode::ControlStateUnavailable, &link.consumer_site, "persist external slot overlay removal", @@ -1199,7 +1309,7 @@ pub(super) async fn clear_external_slot_overlay( ) .await .map_err(|err| { - actuator_protocol_error( + controller_protocol_error( ProtocolErrorCode::ControlStateUnavailable, &link.consumer_site, "retract external slot overlay", @@ -1217,9 +1327,13 @@ pub(super) async fn clear_export_peer_overlay( let overlay_id = overlay_id_for_link_action(child, link, |action| { matches!(action, DynamicOverlayAction::ExportPeer { .. }) })?; - clear_desired_overlay_for_provider(&site_state_root_for(app, &link.provider_site), overlay_id) + app.runtime + .clear_desired_overlay_for_provider( + &site_state_root_for(app, &link.provider_site), + overlay_id, + ) .map_err(|err| { - actuator_protocol_error( + controller_protocol_error( ProtocolErrorCode::ControlStateUnavailable, &link.provider_site, "persist export-peer overlay removal", @@ -1248,7 +1362,7 @@ pub(super) async fn clear_export_peer_overlay( ) .await .map_err(|err| { - actuator_protocol_error( + controller_protocol_error( ProtocolErrorCode::ControlStateUnavailable, &link.provider_site, "retract export-peer overlay", @@ -1277,6 +1391,84 @@ pub(super) async fn retract_child_overlays( Ok(()) } +pub(super) async fn retract_dynamic_capability_origin_overlays( + app: &ControlStateApp, + child: &LiveChildRecord, +) -> std::result::Result<(), ProtocolErrorResponse> { + let overlays_by_site = { + let state = app.control_state.lock().await; + let Some(fragment) = child.fragment.as_ref() else { + return Ok(()); + }; + let mut overlays_by_site = BTreeMap::>::new(); + for component in &fragment.components { + if component.program.is_none() { + continue; + } + let holder_component_id = dynamic_caps::logical_component_id(&component.moniker); + for held in dynamic_caps::live_held_entries(&state, &holder_component_id)? { + let selector = if let Some(selector) = held.root_authority_selector.clone() { + selector + } else if let Some(grant_id) = held.grant_id.as_deref() { + state + .dynamic_capability_grants + .get(grant_id) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("dynamic grant `{grant_id}` is missing from state"), + ) + })? + .root_authority_selector + .clone() + } else { + continue; + }; + let site_id = site_id_for_root_authority_selector(&state, &selector)?; + overlays_by_site.entry(site_id).or_default().insert( + dynamic_caps::origin_overlay_id(&holder_component_id, &selector), + ); + } + } + overlays_by_site + }; + + for (site_id, overlay_ids) in overlays_by_site { + let router_control = load_site_manager_state(app, &site_id)? + .router_control + .ok_or_else(|| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &site_id, + "retract dynamic capability origin overlays", + "site router control endpoint is unavailable", + ) + })?; + let endpoint = parse_control_endpoint(&router_control).map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &site_id, + "retract dynamic capability origin overlays", + format!("site router control endpoint is invalid: {err}"), + ) + })?; + for overlay_id in overlay_ids { + revoke_route_overlay_with_retry(&endpoint, &overlay_id, Duration::from_secs(30)) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &site_id, + "retract dynamic capability origin overlays", + err, + ) + })?; + } + } + + Ok(()) +} + pub(super) fn child_site_publish_waves(child: &LiveChildRecord) -> Vec> { let site_ids = child .site_plans @@ -1411,6 +1603,7 @@ pub(super) async fn continue_create_committed_hidden( let child_sites = site_plans.keys().cloned().collect::>(); let links = child_link_records(&child); let mut published_child_sites = BTreeSet::new(); + let mut published_links = Vec::new(); for wave in child_site_publish_waves(&child) { for site_id in &wave { for link in links.iter().filter(|link| link.consumer_site == *site_id) { @@ -1418,6 +1611,7 @@ pub(super) async fn continue_create_committed_hidden( || published_child_sites.contains(&link.provider_site); if provider_ready { publish_link_overlays(app, &child, link).await?; + published_links.push(link.clone()); } } } @@ -1453,6 +1647,66 @@ pub(super) async fn continue_create_committed_hidden( } } if let Some(err) = first_error { + let desired_site_plans = { + let state = app.control_state.lock().await; + desired_site_plan_map(&state, &child_sites)? + }; + let mut cleanup_error = None; + for link in published_links.iter().rev() { + if let Err(retract_err) = retract_link_overlays(app, &child, link).await + && cleanup_error.is_none() + { + cleanup_error = Some(retract_err); + } + } + + let mut destroy_tasks = tokio::task::JoinSet::new(); + for site_id in &child_sites { + let app = app.clone(); + let desired_site_plan = desired_site_plans.get(site_id).cloned(); + let site_id = site_id.clone(); + destroy_tasks.spawn(async move { + destroy_child_on_site(&app, child.child_id, &site_id, desired_site_plan).await + }); + } + while let Some(result) = destroy_tasks.join_next().await { + match result { + Ok(Ok(())) => {} + Ok(Err(destroy_err)) if cleanup_error.is_none() => { + cleanup_error = Some(destroy_err); + } + Ok(Err(_)) => {} + Err(join_err) if cleanup_error.is_none() => { + cleanup_error = Some(protocol_error( + ProtocolErrorCode::PublishFailed, + &format!("site destroy task failed: {join_err}"), + )); + } + Err(_) => {} + } + } + + if let Some(cleanup_error) = cleanup_error { + return Err(protocol_error( + ProtocolErrorCode::PublishFailed, + &format!("{}; cleanup failed: {}", err.message, cleanup_error.message), + )); + } + + let mut state = app.control_state.lock().await; + if child_record_location(&state, child.child_id).is_ok() { + let tx_id = child_create_tx_id(&state, child.child_id)?; + persist_control_state_update( + &mut state, + &app.state_path, + "create_aborted", + |state| { + append_journal_entry(state, tx_id, &child, ChildState::CreateAborted); + remove_child_record(state, child.child_id)?; + Ok(()) + }, + )?; + } return Err(err); } for site_id in published_wave_sites { @@ -1553,6 +1807,7 @@ pub(super) async fn continue_destroy_requested( child }; retract_child_overlays(app, &child).await?; + retract_dynamic_capability_origin_overlays(app, &child).await?; { let mut state = app.control_state.lock().await; diff --git a/cli/src/framework_component/planner.rs b/runtime/site-controller/src/planner.rs similarity index 99% rename from cli/src/framework_component/planner.rs rename to runtime/site-controller/src/planner.rs index 3581b969..f7787472 100644 --- a/cli/src/framework_component/planner.rs +++ b/runtime/site-controller/src/planner.rs @@ -1154,12 +1154,12 @@ pub(super) fn wrapper_manifest_url(authority_realm_id: usize, child_id: u64) -> } pub(super) fn allocate_child_id(state: &mut FrameworkControlState) -> u64 { - state.next_child_id += 1; + state.next_child_id += state.id_stride.max(1); state.next_child_id } pub(super) fn allocate_tx_id(state: &mut FrameworkControlState) -> u64 { - state.next_tx_id += 1; + state.next_tx_id += state.id_stride.max(1); state.next_tx_id } @@ -3048,17 +3048,22 @@ pub(super) struct ControlStateApp { pub(super) state_root: PathBuf, pub(super) mesh_scope: Arc, pub(super) control_state_auth_token: Arc, + pub(super) controller_plan: Arc, + pub(super) peer_controllers: Arc>, pub(super) authority_locks: Arc>>>>, - pub(super) bridge_proxies: Arc>>, + pub(super) runtime: SharedSiteControllerRuntime, } #[derive(Clone)] -pub(super) struct CcsApp { - pub(super) client: ReqwestClient, - pub(super) site_state_root: PathBuf, - pub(super) control_state_url: Arc, +pub(super) struct SiteControllerApp { + pub(super) control: ControlStateApp, pub(super) router_auth_token: Arc, - pub(super) control_state_auth_token: Arc, +} + +#[derive(Clone)] +pub(super) struct LocalDynamicCapabilityOriginApp { + pub(super) site_state_root: PathBuf, + pub(super) runtime: SharedSiteControllerRuntime, } #[derive(Clone, Debug, Deserialize)] @@ -3085,4 +3090,8 @@ pub(super) struct SiteManagerStateView { pub(super) router_identity_id: Option, #[serde(default)] pub(super) router_public_key_b64: Option, + #[serde(default)] + pub(super) site_controller_pid: Option, + #[serde(default)] + pub(super) site_controller_url: Option, } diff --git a/runtime/site-controller/src/runtime_api.rs b/runtime/site-controller/src/runtime_api.rs new file mode 100644 index 00000000..e796ec02 --- /dev/null +++ b/runtime/site-controller/src/runtime_api.rs @@ -0,0 +1,458 @@ +use std::{ + collections::BTreeMap, + future::Future, + net::SocketAddr, + path::{Path, PathBuf}, + pin::Pin, + sync::Arc, +}; + +use amber_compiler::run_plan::{RunLink, SiteKind}; +use amber_mesh::{MeshConfigPublic, MeshIdentityPublic}; +use amber_proxy::ControlEndpoint; +use base64::Engine as _; +use miette::{IntoDiagnostic as _, Result, WrapErr as _}; +use serde::{Deserialize, Serialize}; + +use super::state::{DynamicSitePlanRecord, SiteControllerPlan}; + +const KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH: &str = "01-configmaps/amber-mesh-provision.yaml"; +const KUBERNETES_PROVISIONER_JOB_PATH: &str = "02-rbac/amber-provisioner-job.yaml"; +const KUBERNETES_PROVISIONER_ROLE_PATH: &str = "02-rbac/amber-provisioner-role.yaml"; +const KUBERNETES_PROVISIONER_ROLEBINDING_PATH: &str = "02-rbac/amber-provisioner-rolebinding.yaml"; +const KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH: &str = "02-rbac/amber-provisioner-sa.yaml"; +const KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME: &str = "amber-router-external"; +const SITE_CONTROLLER_RUNTIME_PLAN_SCHEMA: &str = "amber.run.site_controller_runtime_plan"; +const SITE_CONTROLLER_RUNTIME_PLAN_VERSION: u32 = 1; + +pub type SiteControllerRuntimeFuture<'a, T> = Pin> + Send + 'a>>; + +pub trait SiteControllerRuntime: Send + Sync { + fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()>; + + fn resolve_link_external_url<'a>( + &'a self, + provider: &'a LaunchedSite, + provider_output_dir: &'a Path, + link: &'a RunLink, + consumer_kind: SiteKind, + run_root: &'a Path, + ) -> SiteControllerRuntimeFuture<'a, String>; + + fn prepare_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + child_id: u64, + site_plan: DynamicSitePlanRecord, + ) -> SiteControllerRuntimeFuture<'a, ()>; + + fn publish_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + child_id: u64, + site_plan: DynamicSitePlanRecord, + ) -> SiteControllerRuntimeFuture<'a, ()>; + + fn rollback_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + child_id: u64, + ) -> SiteControllerRuntimeFuture<'a, ()>; + + fn destroy_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + child_id: u64, + desired_site_plan: Option, + ) -> SiteControllerRuntimeFuture<'a, ()>; + + fn collect_live_component_runtime_metadata( + &self, + plan: &SiteControllerRuntimePlan, + ) -> Result>; + + fn load_live_site_router_mesh_config( + &self, + plan: &SiteControllerRuntimePlan, + ) -> Result; + + fn router_mesh_addr_for_consumer( + &self, + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> Result; + + fn update_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExternalSlotOverlay, + ) -> Result<()>; + + fn update_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExportPeerOverlay, + ) -> Result<()>; + + fn clear_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> Result<()>; + + fn clear_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> Result<()>; +} + +pub type SharedSiteControllerRuntime = Arc; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SiteReceipt { + pub kind: SiteKind, + pub artifact_dir: String, + pub supervisor_pid: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub process_pid: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub compose_project: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub kubernetes_namespace: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub port_forward_pid: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub router_control: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub router_identity_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub router_public_key_b64: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub site_controller_pid: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub site_controller_url: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SiteControllerRuntimePlan { + pub schema: String, + pub version: u32, + pub run_id: String, + pub mesh_scope: String, + pub run_root: String, + pub site_id: String, + pub kind: SiteKind, + pub router_identity_id: String, + pub artifact_dir: String, + pub site_state_root: String, + pub listen_addr: SocketAddr, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage_root: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub runtime_root: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub router_mesh_port: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub compose_project: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub kubernetes_namespace: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observability_endpoint: Option, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub launch_env: BTreeMap, +} + +#[derive(Clone, Debug)] +pub struct LiveComponentRuntimeMetadata { + pub moniker: String, + pub host_mesh_addr: String, + pub mesh_config: MeshConfigPublic, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DesiredExternalSlotOverlay { + pub slot_name: String, + pub url: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DesiredExportPeerOverlay { + pub export_name: String, + pub peer_id: String, + pub peer_key_b64: String, + pub protocol: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub route_id: Option, +} + +#[derive(Clone, Debug)] +pub struct LaunchedSite { + pub receipt: SiteReceipt, + pub router_control: ControlEndpoint, + pub router_identity: MeshIdentityPublic, + pub router_addr: SocketAddr, +} + +pub fn site_controller_runtime_plan_from_controller_plan( + plan: &SiteControllerPlan, +) -> SiteControllerRuntimePlan { + SiteControllerRuntimePlan { + schema: SITE_CONTROLLER_RUNTIME_PLAN_SCHEMA.to_string(), + version: SITE_CONTROLLER_RUNTIME_PLAN_VERSION, + run_id: plan.run_id.clone(), + mesh_scope: plan.mesh_scope.clone(), + run_root: plan.run_root.clone(), + site_id: plan.site_id.clone(), + kind: plan.kind, + router_identity_id: plan.router_identity_id.clone(), + artifact_dir: plan.artifact_dir.clone(), + site_state_root: plan.site_state_root.clone(), + listen_addr: plan.listen_addr, + storage_root: plan.storage_root.clone(), + runtime_root: plan.runtime_root.clone(), + router_mesh_port: plan.router_mesh_port, + compose_project: plan.compose_project.clone(), + kubernetes_namespace: plan.kubernetes_namespace.clone(), + context: plan.context.clone(), + observability_endpoint: plan.observability_endpoint.clone(), + launch_env: plan.launch_env.clone(), + } +} + +pub fn site_controller_runtime_child_root_for_site( + site_state_root: &Path, + child_id: u64, +) -> PathBuf { + site_state_root + .join("framework-component") + .join("children") + .join(child_id.to_string()) +} + +pub fn site_state_path(state_root: &Path, site_id: &str) -> PathBuf { + state_root.join(site_id).join("manager-state.json") +} + +pub fn site_controller_plan_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("site-controller-plan.json") +} + +fn yaml_string(value: &str) -> serde_yaml::Value { + serde_yaml::Value::String(value.to_string()) +} + +fn kubernetes_resource_name(document: &serde_yaml::Value) -> Option<&str> { + document + .as_mapping() + .and_then(|root| root.get(yaml_string("metadata"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|metadata| metadata.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) +} + +fn kubernetes_dynamic_apply_resource_kept_from_contents( + resource: &str, + raw: &str, + child_component_labels: &std::collections::BTreeSet, +) -> Result { + if matches!( + resource, + KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH + | KUBERNETES_PROVISIONER_JOB_PATH + | KUBERNETES_PROVISIONER_ROLE_PATH + | KUBERNETES_PROVISIONER_ROLEBINDING_PATH + | KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH + ) || resource.starts_with("03-persistentvolumeclaims/") + { + return Ok(true); + } + + let document: serde_yaml::Value = serde_yaml::from_str(raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid kubernetes resource {resource}"))?; + Ok(document + .as_mapping() + .and_then(|root| root.get(yaml_string("metadata"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|metadata| metadata.get(yaml_string("labels"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|labels| labels.get(yaml_string("amber.io/component-id"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|component_id| child_component_labels.contains(component_id))) +} + +pub fn project_kubernetes_dynamic_child_artifact_files( + artifact_files: &BTreeMap, + component_ids: &[usize], +) -> Result> { + let child_component_labels = component_ids + .iter() + .map(|component_id| format!("c{component_id}")) + .collect::>(); + let kustomization_path = "kustomization.yaml"; + let raw = artifact_files.get(kustomization_path).ok_or_else(|| { + miette::miette!("dynamic kubernetes artifact snapshot is missing {kustomization_path}") + })?; + let mut document: serde_yaml::Value = serde_yaml::from_str(raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid kustomization {kustomization_path}"))?; + let root = document.as_mapping_mut().ok_or_else(|| { + miette::miette!("kustomization {kustomization_path} is not a YAML mapping") + })?; + let resources = root + .get_mut(yaml_string("resources")) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!("kustomization {kustomization_path} is missing a resources sequence") + })?; + let mut projected = artifact_files + .iter() + .filter(|(path, _)| !path.ends_with(".yaml") && path.as_str() != kustomization_path) + .map(|(path, contents)| (path.clone(), contents.clone())) + .collect::>(); + let mut kept_resources = Vec::new(); + let mut kept_resource_names = std::collections::BTreeSet::new(); + for resource in resources + .iter() + .filter_map(serde_yaml::Value::as_str) + .map(str::to_owned) + { + let raw = artifact_files + .get(&resource) + .ok_or_else(|| miette::miette!("dynamic kubernetes artifact is missing {resource}"))?; + if !kubernetes_dynamic_apply_resource_kept_from_contents( + &resource, + raw, + &child_component_labels, + )? { + continue; + } + let document: serde_yaml::Value = serde_yaml::from_str(raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid kubernetes resource {resource}"))?; + if let Some(name) = kubernetes_resource_name(&document) { + kept_resource_names.insert(name.to_string()); + } + projected.insert(resource.clone(), raw.clone()); + kept_resources.push(serde_yaml::Value::String(resource)); + } + *resources = kept_resources; + + if let Some(generators) = root + .get_mut(yaml_string("secretGenerator")) + .and_then(serde_yaml::Value::as_sequence_mut) + { + generators.retain(|generator| { + generator + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + != Some(KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME) + }); + } + + if let Some(replacements) = root + .get_mut(yaml_string("replacements")) + .and_then(serde_yaml::Value::as_sequence_mut) + { + replacements.retain_mut(|replacement| { + let Some(targets) = replacement + .as_mapping_mut() + .and_then(|mapping| mapping.get_mut(yaml_string("targets"))) + .and_then(serde_yaml::Value::as_sequence_mut) + else { + return false; + }; + targets.retain(|target| { + target + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("select"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|select| select.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|name| kept_resource_names.contains(name)) + }); + !targets.is_empty() + }); + } + + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {kustomization_path}"))?; + projected.insert(kustomization_path.to_string(), rendered); + Ok(projected) +} + +pub fn parse_control_endpoint(raw: &str) -> Result { + if let Some(path) = raw.strip_prefix("unix://") { + return Ok(ControlEndpoint::Unix(PathBuf::from(path))); + } + if let Some(rest) = raw.strip_prefix("volume://") + && let Some((volume, socket_path)) = rest.split_once('/') + { + return Ok(ControlEndpoint::VolumeSocket { + volume: volume.to_string(), + socket_path: format!("/{}", socket_path), + }); + } + Ok(ControlEndpoint::Tcp(raw.to_string())) +} + +pub fn decode_public_key(value: &str) -> Result<[u8; 32]> { + let bytes = base64::engine::general_purpose::STANDARD + .decode(value) + .into_diagnostic() + .wrap_err("invalid base64 router public key")?; + bytes + .as_slice() + .try_into() + .map_err(|_| miette::miette!("invalid router public key length")) +} + +pub fn launched_site_from_receipt( + site_receipt: &SiteReceipt, + mesh_scope: &str, +) -> Result { + let router_control = parse_control_endpoint( + site_receipt + .router_control + .as_deref() + .ok_or_else(|| miette::miette!("site receipt is missing router control"))?, + )?; + let router_addr = site_receipt + .router_mesh_addr + .as_deref() + .ok_or_else(|| miette::miette!("site receipt is missing router mesh addr"))? + .parse() + .into_diagnostic() + .wrap_err("invalid router mesh addr in site receipt")?; + let router_identity = MeshIdentityPublic { + id: site_receipt + .router_identity_id + .clone() + .ok_or_else(|| miette::miette!("site receipt is missing router identity id"))?, + public_key: decode_public_key( + site_receipt + .router_public_key_b64 + .as_deref() + .ok_or_else(|| miette::miette!("site receipt is missing router public key"))?, + )?, + mesh_scope: Some(mesh_scope.to_string()), + }; + Ok(LaunchedSite { + receipt: site_receipt.clone(), + router_control, + router_identity, + router_addr, + }) +} diff --git a/runtime/site-controller/src/site_controller.rs b/runtime/site-controller/src/site_controller.rs new file mode 100644 index 00000000..7a338083 --- /dev/null +++ b/runtime/site-controller/src/site_controller.rs @@ -0,0 +1,1202 @@ +use std::sync::Arc; + +use amber_mesh::dynamic_caps::HeldListResponse; +use axum::{ + Json, Router, + extract::{Path as AxumPath, State}, + http::{HeaderMap, Method, StatusCode}, + routing::{get, post}, +}; +use miette::{IntoDiagnostic as _, Result, WrapErr as _}; +use reqwest::Client as ReqwestClient; +use serde::{Serialize, de::DeserializeOwned}; +use tokio::net::TcpListener; + +use super::{ + ccs_api::{self, FrameworkComponentInspectRequest, FrameworkComponentInspectResponse}, + control_state_api::{ + self, DynamicCapsInspectRequest, DynamicCapsInspectResponse, DynamicCapsMutateRequest, + DynamicCapsMutateResponse, + }, + dynamic_caps::{ + self, ControlDynamicHeldDetailRequest, ControlDynamicHeldListRequest, + ControlDynamicInspectRefRequest, ControlDynamicResolveOriginRequest, + ControlDynamicRevokeRequest, ControlDynamicShareRequest, + }, + http::{ + authorize_framework_auth_header, cleanup_dynamic_bridge_proxies, healthz, read_json, + required_header, shutdown_signal, + }, + orchestration::{ + ControlCreateChildRequest, ControlDestroyChildRequest, ProtocolApiError, + SiteControllerDestroyRequest, SiteControllerPrepareRequest, SiteControllerPublishRequest, + execute_create_child, execute_destroy_child, publish_dynamic_capability_origin_local, + recover_control_state, + }, + planner::{ + ControlStateApp, LocalDynamicCapabilityOriginApp, SiteControllerApp, protocol_error, + }, + state::*, + *, +}; +use crate::api::capability_instance_record; + +const CONTROLLER_LOCAL_ONLY_HEADER: &str = "x-amber-site-controller-local-only"; + +pub async fn run_site_controller( + plan_path: PathBuf, + runtime: SharedSiteControllerRuntime, +) -> Result<()> { + let plan: SiteControllerPlan = read_json(plan_path.as_path(), "site controller plan")?; + let mut control_state: FrameworkControlState = + read_json(Path::new(&plan.state_path), "site controller state file")?; + persist_control_state(Path::new(&plan.state_path), &mut control_state)?; + let control = ControlStateApp { + control_state: Arc::new(Mutex::new(control_state)), + client: ReqwestClient::new(), + state_path: PathBuf::from(&plan.state_path), + run_root: PathBuf::from(&plan.run_root), + state_root: PathBuf::from(&plan.state_root), + mesh_scope: Arc::::from(plan.mesh_scope.clone()), + control_state_auth_token: Arc::::from(plan.auth_token.clone()), + controller_plan: Arc::new(plan.clone()), + peer_controllers: Arc::new(plan.peer_controllers.clone()), + authority_locks: Arc::new(Mutex::new(BTreeMap::new())), + runtime, + }; + let app_state = SiteControllerApp { + control: control.clone(), + router_auth_token: Arc::::from(plan.auth_token), + }; + recover_control_state(&control).await?; + let app = site_controller_router(app_state.clone()); + let listener = TcpListener::bind(plan.listen_addr) + .await + .into_diagnostic() + .wrap_err_with(|| format!("failed to bind site controller on {}", plan.listen_addr))?; + let serve_result = axum::serve(listener, app.into_make_service()) + .with_graceful_shutdown(shutdown_signal()) + .await + .into_diagnostic(); + let cleanup_result = cleanup_dynamic_bridge_proxies(&control).await; + match (serve_result, cleanup_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(err), Ok(())) => Err(err).wrap_err("site controller failed"), + (Ok(()), Err(err)) => Err(err).wrap_err("site controller failed to stop bridge proxies"), + (Err(serve_err), Err(cleanup_err)) => Err(miette::miette!( + "site controller failed: {serve_err}\nbridge proxy cleanup failed: {cleanup_err}" + )), + } +} + +pub(crate) fn site_controller_router(app_state: SiteControllerApp) -> Router { + Router::new() + .nest_service("/mcp", super::mcp::service(app_state.clone())) + .nest_service( + "/v1/controller/dynamic-caps/mcp", + super::control_state_mcp::service(app_state.clone()), + ) + .route("/", get(healthz)) + .route("/healthz", get(healthz)) + .route( + SITE_CONTROLLER_STATE_PATH, + get(get_site_controller_state_route), + ) + .route("/v1/templates", get(list_templates)) + .route("/v1/templates/{template}", get(describe_template)) + .route("/v1/templates/{template}/resolve", post(resolve_template)) + .route("/v1/children", get(list_children).post(create_child)) + .route( + "/v1/children/{child}", + get(describe_child).delete(destroy_child), + ) + .route("/v1/snapshot", post(snapshot)) + .route( + "/v1/internal/dynamic-caps/origins/publish", + post(publish_dynamic_origin), + ) + .route("/v1/controller/children", post(control_create_child_route)) + .route( + "/v1/controller/children/{child}/destroy", + post(control_destroy_child_route), + ) + .route( + "/v1/controller/dynamic-caps/held", + post(control_dynamic_held_list_route), + ) + .route( + "/v1/controller/dynamic-caps/held/detail", + post(control_dynamic_held_detail_route), + ) + .route( + "/v1/controller/dynamic-caps/share", + post(control_dynamic_share_route), + ) + .route( + "/v1/controller/dynamic-caps/inspect-ref", + post(control_dynamic_inspect_ref_route), + ) + .route( + "/v1/controller/dynamic-caps/revoke", + post(control_dynamic_revoke_route), + ) + .route( + "/v1/controller/dynamic-caps/resolve-origin", + post(control_dynamic_resolve_origin_route), + ) + .route( + "/v1/controller/site/children/{child_id}/prepare", + post(site_prepare_child_route), + ) + .route( + "/v1/controller/site/children/{child_id}/publish", + post(site_publish_child_route), + ) + .route( + "/v1/controller/site/children/{child_id}/rollback", + post(site_rollback_child_route), + ) + .route( + "/v1/controller/site/children/{child_id}/destroy", + post(site_destroy_child_route), + ) + .with_state(app_state) +} + +async fn get_site_controller_state_route( + State(app): State, + headers: HeaderMap, +) -> std::result::Result, ProtocolApiError> { + authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + Ok(Json(app.control.control_state.lock().await.clone())) +} + +pub(super) async fn authorize_public_request( + app: &SiteControllerApp, + headers: &HeaderMap, +) -> std::result::Result<(CapabilityInstanceRecord, FrameworkControlState), ProtocolApiError> { + authorize_framework_auth_header(headers, app.router_auth_token.as_ref())?; + let route_id = required_header(headers, FRAMEWORK_ROUTE_ID_HEADER)?; + let peer_id = required_header(headers, FRAMEWORK_PEER_ID_HEADER)?; + let state = app.control.control_state.lock().await.clone(); + let record = super::api::authorize_capability_instance(&state, &route_id, &peer_id) + .map_err(ProtocolApiError::from)? + .clone(); + Ok((record, state)) +} + +fn controller_local_only(headers: &HeaderMap) -> bool { + headers + .get(CONTROLLER_LOCAL_ONLY_HEADER) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value == "1") +} + +fn peer_controller_url<'a>( + app: &'a SiteControllerApp, + site_id: &str, +) -> std::result::Result<&'a str, ProtocolApiError> { + if site_id == app.control.controller_plan.site_id { + return Ok(app.control.controller_plan.authority_url.as_str()); + } + app.control + .peer_controllers + .get(site_id) + .map(|plan| plan.authority_url.as_str()) + .ok_or_else(|| { + ProtocolApiError::control_state_unavailable(format!( + "site `{site_id}` controller metadata is unavailable" + )) + }) +} + +async fn forward_framework_request( + app: &SiteControllerApp, + site_id: &str, + method: Method, + path: &str, + record: &CapabilityInstanceRecord, + body: Option<&TReq>, +) -> std::result::Result { + let mut request = app + .control + .client + .request( + reqwest::Method::from_bytes(method.as_str().as_bytes()) + .expect("axum HTTP method should map to reqwest"), + format!( + "{}{}", + peer_controller_url(app, site_id)?.trim_end_matches('/'), + path + ), + ) + .header(FRAMEWORK_AUTH_HEADER, app.router_auth_token.as_ref()) + .header(FRAMEWORK_ROUTE_ID_HEADER, record.route_id.as_str()) + .header(FRAMEWORK_PEER_ID_HEADER, record.recipient_peer_id.as_str()); + if let Some(body) = body { + request = request.json(body); + } + let response = request.send().await.map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "failed to reach site controller for site `{site_id}`: {err}" + )) + })?; + if response.status().is_success() { + return response.json().await.map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "site controller for site `{site_id}` returned invalid JSON: {err}" + )) + }); + } + let status = response.status(); + let body = response.bytes().await.map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "failed to read site controller error response for site `{site_id}`: {err}" + )) + })?; + if let Ok(protocol_error) = serde_json::from_slice::(&body) { + return Err(protocol_error.into()); + } + Err(ProtocolApiError::control_state_unavailable(format!( + "site controller for site `{site_id}` returned {status}" + ))) +} + +async fn forward_framework_empty( + app: &SiteControllerApp, + site_id: &str, + method: Method, + path: &str, + record: &CapabilityInstanceRecord, + body: Option<&TReq>, +) -> std::result::Result<(), ProtocolApiError> { + let mut request = app + .control + .client + .request( + reqwest::Method::from_bytes(method.as_str().as_bytes()) + .expect("axum HTTP method should map to reqwest"), + format!( + "{}{}", + peer_controller_url(app, site_id)?.trim_end_matches('/'), + path + ), + ) + .header(FRAMEWORK_AUTH_HEADER, app.router_auth_token.as_ref()) + .header(FRAMEWORK_ROUTE_ID_HEADER, record.route_id.as_str()) + .header(FRAMEWORK_PEER_ID_HEADER, record.recipient_peer_id.as_str()); + if let Some(body) = body { + request = request.json(body); + } + let response = request.send().await.map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "failed to reach site controller for site `{site_id}`: {err}" + )) + })?; + if response.status().is_success() { + return Ok(()); + } + let status = response.status(); + let body = response.bytes().await.map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "failed to read site controller error response for site `{site_id}`: {err}" + )) + })?; + if let Ok(protocol_error) = serde_json::from_slice::(&body) { + return Err(protocol_error.into()); + } + Err(ProtocolApiError::control_state_unavailable(format!( + "site controller for site `{site_id}` returned {status}" + ))) +} + +async fn peer_dynamic_caps_post( + app: &SiteControllerApp, + site_id: &str, + path: &str, + body: &TReq, +) -> std::result::Result { + let response = app + .control + .client + .post(format!( + "{}{}", + peer_controller_url(app, site_id)?.trim_end_matches('/'), + path + )) + .header( + FRAMEWORK_AUTH_HEADER, + app.control.control_state_auth_token.as_ref(), + ) + .header(CONTROLLER_LOCAL_ONLY_HEADER, "1") + .json(body) + .send() + .await + .map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "failed to reach site controller for site `{site_id}`: {err}" + )) + })?; + if response.status().is_success() { + return response.json().await.map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "site controller for site `{site_id}` returned invalid JSON: {err}" + )) + }); + } + let status = response.status(); + let body = response.bytes().await.map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "failed to read site controller error response for site `{site_id}`: {err}" + )) + })?; + if let Ok(protocol_error) = serde_json::from_slice::(&body) { + return Err(protocol_error.into()); + } + Err(ProtocolApiError::control_state_unavailable(format!( + "site controller for site `{site_id}` returned {status}" + ))) +} + +async fn local_held_list( + app: &SiteControllerApp, + request: ControlDynamicHeldListRequest, +) -> std::result::Result { + match control_state_api::execute_dynamic_caps_inspect( + &app.control, + DynamicCapsInspectRequest::HeldList(request), + ) + .await? + { + DynamicCapsInspectResponse::HeldList(response) => Ok(response), + _ => unreachable!("held_list should return held list"), + } +} + +async fn local_held_detail( + app: &SiteControllerApp, + request: ControlDynamicHeldDetailRequest, +) -> std::result::Result { + match control_state_api::execute_dynamic_caps_inspect( + &app.control, + DynamicCapsInspectRequest::HeldDetail(request), + ) + .await? + { + DynamicCapsInspectResponse::HeldDetail(response) => Ok(response), + _ => unreachable!("held_detail should return held detail"), + } +} + +async fn local_inspect_ref( + app: &SiteControllerApp, + request: ControlDynamicInspectRefRequest, +) -> std::result::Result { + match control_state_api::execute_dynamic_caps_inspect( + &app.control, + DynamicCapsInspectRequest::InspectRef(request), + ) + .await? + { + DynamicCapsInspectResponse::InspectRef(response) => Ok(response), + _ => unreachable!("inspect_ref should return inspect response"), + } +} + +async fn local_resolve_origin( + app: &SiteControllerApp, + request: ControlDynamicResolveOriginRequest, +) -> std::result::Result { + match control_state_api::execute_dynamic_caps_inspect( + &app.control, + DynamicCapsInspectRequest::ResolveOrigin(request), + ) + .await? + { + DynamicCapsInspectResponse::ResolveOrigin(response) => Ok(response), + _ => unreachable!("resolve_origin should return origin resolution"), + } +} + +async fn local_share( + app: &SiteControllerApp, + request: ControlDynamicShareRequest, +) -> std::result::Result { + match control_state_api::execute_dynamic_caps_mutate( + &app.control, + DynamicCapsMutateRequest::Share(request), + ) + .await? + { + DynamicCapsMutateResponse::Share(response) => Ok(response), + _ => unreachable!("share should return share response"), + } +} + +async fn local_revoke( + app: &SiteControllerApp, + request: ControlDynamicRevokeRequest, +) -> std::result::Result { + match control_state_api::execute_dynamic_caps_mutate( + &app.control, + DynamicCapsMutateRequest::Revoke(request), + ) + .await? + { + DynamicCapsMutateResponse::Revoke(response) => Ok(response), + _ => unreachable!("revoke should return revoke response"), + } +} + +fn site_for_dynamic_source( + state: &FrameworkControlState, + source: &dynamic_caps::DynamicCapabilityControlSourceRequest, +) -> std::result::Result { + match source { + dynamic_caps::DynamicCapabilityControlSourceRequest::RootAuthority { + root_authority_selector, + } => site_id_for_root_authority_selector(state, root_authority_selector), + dynamic_caps::DynamicCapabilityControlSourceRequest::Grant { grant_id } => { + site_id_for_dynamic_grant(state, grant_id) + } + } + .map_err(ProtocolApiError::from) +} + +fn site_for_held_id( + state: &FrameworkControlState, + held_id: &str, +) -> std::result::Result { + match dynamic_caps::parse_held_entry_key(held_id).map_err(ProtocolApiError::from)? { + dynamic_caps::HeldEntryKey::RootAuthority(selector) => { + site_id_for_root_authority_selector(state, &selector) + } + dynamic_caps::HeldEntryKey::Grant(grant_id) => site_id_for_dynamic_grant(state, &grant_id), + } + .map_err(ProtocolApiError::from) +} + +fn site_for_dynamic_ref( + state: &FrameworkControlState, + raw_ref: &str, +) -> std::result::Result { + let parsed = amber_mesh::dynamic_caps::decode_dynamic_capability_ref_unverified(raw_ref) + .map_err(|err| { + ProtocolApiError::from(protocol_error( + ProtocolErrorCode::MalformedRef, + &format!("dynamic capability ref is malformed: {err}"), + )) + })?; + site_id_for_dynamic_grant(state, &parsed.claims.grant_id).map_err(ProtocolApiError::from) +} + +fn framework_request_site_id( + state: &FrameworkControlState, + record: &CapabilityInstanceRecord, +) -> std::result::Result { + match site_id_for_authority_realm(state, record.authority_realm_id) { + Ok(site_id) => Ok(site_id), + Err(_) => Ok(record.recipient_site_id.clone()), + } +} + +pub(super) async fn execute_site_controller_framework_inspect( + app: &SiteControllerApp, + record: &CapabilityInstanceRecord, + state: &FrameworkControlState, + request: FrameworkComponentInspectRequest, +) -> std::result::Result { + let authority_site = framework_request_site_id(state, record)?; + if authority_site != app.control.controller_plan.site_id { + return match request { + FrameworkComponentInspectRequest::ListTemplates => { + Ok(FrameworkComponentInspectResponse::ListTemplates( + forward_framework_request( + app, + &authority_site, + Method::GET, + "/v1/templates", + record, + None::<&()>, + ) + .await?, + )) + } + FrameworkComponentInspectRequest::GetTemplate { template } => { + Ok(FrameworkComponentInspectResponse::GetTemplate( + forward_framework_request( + app, + &authority_site, + Method::GET, + &format!("/v1/templates/{template}"), + record, + None::<&()>, + ) + .await?, + )) + } + FrameworkComponentInspectRequest::ResolveTemplate { template, request } => { + Ok(FrameworkComponentInspectResponse::ResolveTemplate( + forward_framework_request( + app, + &authority_site, + Method::POST, + &format!("/v1/templates/{template}/resolve"), + record, + Some(&request), + ) + .await?, + )) + } + FrameworkComponentInspectRequest::ListChildren => { + Ok(FrameworkComponentInspectResponse::ListChildren( + forward_framework_request( + app, + &authority_site, + Method::GET, + "/v1/children", + record, + None::<&()>, + ) + .await?, + )) + } + FrameworkComponentInspectRequest::GetChild { child } => { + Ok(FrameworkComponentInspectResponse::GetChild( + forward_framework_request( + app, + &authority_site, + Method::GET, + &format!("/v1/children/{child}"), + record, + None::<&()>, + ) + .await?, + )) + } + FrameworkComponentInspectRequest::GetSnapshot => { + Ok(FrameworkComponentInspectResponse::GetSnapshot( + forward_framework_request( + app, + &authority_site, + Method::POST, + "/v1/snapshot", + record, + None::<&()>, + ) + .await?, + )) + } + }; + } + + ccs_api::execute_framework_component_inspect(state, record.authority_realm_id, request) + .await + .map_err(ProtocolApiError::from) +} + +pub(super) async fn execute_site_controller_framework_mutate( + app: &SiteControllerApp, + record: &CapabilityInstanceRecord, + state: &FrameworkControlState, + request: ccs_api::FrameworkComponentMutateRequest, +) -> std::result::Result { + let authority_site = framework_request_site_id(state, record)?; + if authority_site != app.control.controller_plan.site_id { + return match request { + ccs_api::FrameworkComponentMutateRequest::CreateChild(request) => { + Ok(ccs_api::FrameworkComponentMutateResponse::CreateChild( + forward_framework_request( + app, + &authority_site, + Method::POST, + "/v1/children", + record, + Some(&request), + ) + .await?, + )) + } + ccs_api::FrameworkComponentMutateRequest::DestroyChild { child } => { + forward_framework_empty::<()>( + app, + &authority_site, + Method::DELETE, + &format!("/v1/children/{child}"), + record, + None, + ) + .await?; + Ok(ccs_api::FrameworkComponentMutateResponse::DestroyChild( + ccs_api::DestroyChildResponse { + child, + destroyed: true, + }, + )) + } + }; + } + + match request { + ccs_api::FrameworkComponentMutateRequest::CreateChild(request) => { + Ok(ccs_api::FrameworkComponentMutateResponse::CreateChild( + execute_create_child(&app.control, record.authority_realm_id, request).await?, + )) + } + ccs_api::FrameworkComponentMutateRequest::DestroyChild { child } => { + execute_destroy_child(&app.control, record.authority_realm_id, &child).await?; + Ok(ccs_api::FrameworkComponentMutateResponse::DestroyChild( + ccs_api::DestroyChildResponse { + child, + destroyed: true, + }, + )) + } + } +} + +pub(super) async fn execute_site_controller_dynamic_caps_inspect( + app: &SiteControllerApp, + request: DynamicCapsInspectRequest, + local_only: bool, +) -> std::result::Result { + match request { + DynamicCapsInspectRequest::HeldList(request) => { + if local_only { + return Ok(DynamicCapsInspectResponse::HeldList( + local_held_list(app, request).await?, + )); + } + let mut held = local_held_list(app, request.clone()).await?.held; + for site_id in app.control.peer_controllers.keys() { + held.extend( + peer_dynamic_caps_post::<_, HeldListResponse>( + app, + site_id, + "/v1/controller/dynamic-caps/held", + &request, + ) + .await? + .held, + ); + } + held.sort_by(|left, right| left.held_id.cmp(&right.held_id)); + Ok(DynamicCapsInspectResponse::HeldList(HeldListResponse { + held, + })) + } + DynamicCapsInspectRequest::HeldDetail(request) => { + if local_only { + return Ok(DynamicCapsInspectResponse::HeldDetail( + local_held_detail(app, request).await?, + )); + } + let state = app.control.control_state.lock().await.clone(); + let site_id = site_for_held_id(&state, &request.held_id)?; + if site_id == app.control.controller_plan.site_id { + return Ok(DynamicCapsInspectResponse::HeldDetail( + local_held_detail(app, request).await?, + )); + } + Ok(DynamicCapsInspectResponse::HeldDetail( + peer_dynamic_caps_post( + app, + &site_id, + "/v1/controller/dynamic-caps/held/detail", + &request, + ) + .await?, + )) + } + DynamicCapsInspectRequest::InspectRef(request) => { + if local_only { + return Ok(DynamicCapsInspectResponse::InspectRef( + local_inspect_ref(app, request).await?, + )); + } + let state = app.control.control_state.lock().await.clone(); + let site_id = site_for_dynamic_ref(&state, &request.r#ref)?; + if site_id == app.control.controller_plan.site_id { + return Ok(DynamicCapsInspectResponse::InspectRef( + local_inspect_ref(app, request).await?, + )); + } + Ok(DynamicCapsInspectResponse::InspectRef( + peer_dynamic_caps_post( + app, + &site_id, + "/v1/controller/dynamic-caps/inspect-ref", + &request, + ) + .await?, + )) + } + DynamicCapsInspectRequest::ResolveOrigin(request) => { + if local_only { + return Ok(DynamicCapsInspectResponse::ResolveOrigin( + local_resolve_origin(app, request).await?, + )); + } + let state = app.control.control_state.lock().await.clone(); + let site_id = site_for_dynamic_source(&state, &request.source)?; + if site_id == app.control.controller_plan.site_id { + return Ok(DynamicCapsInspectResponse::ResolveOrigin( + local_resolve_origin(app, request).await?, + )); + } + Ok(DynamicCapsInspectResponse::ResolveOrigin( + peer_dynamic_caps_post( + app, + &site_id, + "/v1/controller/dynamic-caps/resolve-origin", + &request, + ) + .await?, + )) + } + } +} + +pub(super) async fn execute_site_controller_dynamic_caps_mutate( + app: &SiteControllerApp, + request: DynamicCapsMutateRequest, + local_only: bool, +) -> std::result::Result { + match request { + DynamicCapsMutateRequest::Share(request) => { + if local_only { + return Ok(DynamicCapsMutateResponse::Share( + local_share(app, request).await?, + )); + } + let state = app.control.control_state.lock().await.clone(); + let site_id = site_for_dynamic_source(&state, &request.source)?; + if site_id == app.control.controller_plan.site_id { + return Ok(DynamicCapsMutateResponse::Share( + local_share(app, request).await?, + )); + } + Ok(DynamicCapsMutateResponse::Share( + peer_dynamic_caps_post( + app, + &site_id, + "/v1/controller/dynamic-caps/share", + &request, + ) + .await?, + )) + } + DynamicCapsMutateRequest::Revoke(request) => { + if local_only { + return Ok(DynamicCapsMutateResponse::Revoke( + local_revoke(app, request).await?, + )); + } + let state = app.control.control_state.lock().await.clone(); + let site_id = site_for_dynamic_source(&state, &request.target)?; + if site_id == app.control.controller_plan.site_id { + return Ok(DynamicCapsMutateResponse::Revoke( + local_revoke(app, request).await?, + )); + } + Ok(DynamicCapsMutateResponse::Revoke( + peer_dynamic_caps_post( + app, + &site_id, + "/v1/controller/dynamic-caps/revoke", + &request, + ) + .await?, + )) + } + } +} + +async fn list_templates( + State(app): State, + headers: HeaderMap, +) -> std::result::Result, ProtocolApiError> { + let (record, state) = authorize_public_request(&app, &headers).await?; + match execute_site_controller_framework_inspect( + &app, + &record, + &state, + FrameworkComponentInspectRequest::ListTemplates, + ) + .await? + { + FrameworkComponentInspectResponse::ListTemplates(response) => Ok(Json(response)), + _ => unreachable!("list_templates should return template list"), + } +} + +async fn describe_template( + State(app): State, + headers: HeaderMap, + AxumPath(template): AxumPath, +) -> std::result::Result, ProtocolApiError> { + let (record, state) = authorize_public_request(&app, &headers).await?; + match execute_site_controller_framework_inspect( + &app, + &record, + &state, + FrameworkComponentInspectRequest::GetTemplate { template }, + ) + .await? + { + FrameworkComponentInspectResponse::GetTemplate(response) => Ok(Json(response)), + _ => unreachable!("get_template should return template description"), + } +} + +async fn resolve_template( + State(app): State, + headers: HeaderMap, + AxumPath(template): AxumPath, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + let (record, state) = authorize_public_request(&app, &headers).await?; + match execute_site_controller_framework_inspect( + &app, + &record, + &state, + FrameworkComponentInspectRequest::ResolveTemplate { template, request }, + ) + .await? + { + FrameworkComponentInspectResponse::ResolveTemplate(response) => Ok(Json(response)), + _ => unreachable!("resolve_template should return resolved template"), + } +} + +async fn list_children( + State(app): State, + headers: HeaderMap, +) -> std::result::Result, ProtocolApiError> { + let (record, state) = authorize_public_request(&app, &headers).await?; + match execute_site_controller_framework_inspect( + &app, + &record, + &state, + FrameworkComponentInspectRequest::ListChildren, + ) + .await? + { + FrameworkComponentInspectResponse::ListChildren(response) => Ok(Json(response)), + _ => unreachable!("list_children should return child list"), + } +} + +async fn create_child( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + let (record, state) = authorize_public_request(&app, &headers).await?; + match execute_site_controller_framework_mutate( + &app, + &record, + &state, + ccs_api::FrameworkComponentMutateRequest::CreateChild(request), + ) + .await? + { + ccs_api::FrameworkComponentMutateResponse::CreateChild(response) => Ok(Json(response)), + _ => unreachable!("create_child should return create response"), + } +} + +async fn describe_child( + State(app): State, + headers: HeaderMap, + AxumPath(child): AxumPath, +) -> std::result::Result, ProtocolApiError> { + let (record, state) = authorize_public_request(&app, &headers).await?; + match execute_site_controller_framework_inspect( + &app, + &record, + &state, + FrameworkComponentInspectRequest::GetChild { child }, + ) + .await? + { + FrameworkComponentInspectResponse::GetChild(response) => Ok(Json(response)), + _ => unreachable!("get_child should return child description"), + } +} + +async fn snapshot( + State(app): State, + headers: HeaderMap, +) -> std::result::Result, ProtocolApiError> { + let (record, state) = authorize_public_request(&app, &headers).await?; + match execute_site_controller_framework_inspect( + &app, + &record, + &state, + FrameworkComponentInspectRequest::GetSnapshot, + ) + .await? + { + FrameworkComponentInspectResponse::GetSnapshot(response) => Ok(Json(response)), + _ => unreachable!("get_snapshot should return snapshot"), + } +} + +async fn destroy_child( + State(app): State, + headers: HeaderMap, + AxumPath(child): AxumPath, +) -> std::result::Result { + let (record, state) = authorize_public_request(&app, &headers).await?; + let _ = execute_site_controller_framework_mutate( + &app, + &record, + &state, + ccs_api::FrameworkComponentMutateRequest::DestroyChild { child }, + ) + .await?; + Ok(StatusCode::NO_CONTENT) +} + +async fn publish_dynamic_origin( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> +{ + authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + let ccs_app = LocalDynamicCapabilityOriginApp { + site_state_root: PathBuf::from(&app.control.controller_plan.site_state_root), + runtime: app.control.runtime.clone(), + }; + Ok(Json( + publish_dynamic_capability_origin_local(&ccs_app, request).await?, + )) +} + +async fn control_create_child_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + let authority_realm_id = { + let state = app.control.control_state.lock().await; + capability_instance_record(&state, &request.cap_instance_id) + .map_err(ProtocolApiError::from)? + .authority_realm_id + }; + Ok(Json( + execute_create_child(&app.control, authority_realm_id, request.request).await?, + )) +} + +async fn control_destroy_child_route( + State(app): State, + headers: HeaderMap, + AxumPath(child): AxumPath, + Json(request): Json, +) -> std::result::Result { + authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + let authority_realm_id = { + let state = app.control.control_state.lock().await; + capability_instance_record(&state, &request.cap_instance_id) + .map_err(ProtocolApiError::from)? + .authority_realm_id + }; + execute_destroy_child(&app.control, authority_realm_id, &child).await?; + Ok(StatusCode::NO_CONTENT) +} + +async fn control_dynamic_held_list_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + match execute_site_controller_dynamic_caps_inspect( + &app, + DynamicCapsInspectRequest::HeldList(request), + controller_local_only(&headers), + ) + .await? + { + DynamicCapsInspectResponse::HeldList(response) => Ok(Json(response)), + _ => unreachable!("held_list should return held list"), + } +} + +async fn control_dynamic_held_detail_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + match execute_site_controller_dynamic_caps_inspect( + &app, + DynamicCapsInspectRequest::HeldDetail(request), + controller_local_only(&headers), + ) + .await? + { + DynamicCapsInspectResponse::HeldDetail(response) => Ok(Json(response)), + _ => unreachable!("held_detail should return held detail"), + } +} + +async fn control_dynamic_share_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + match execute_site_controller_dynamic_caps_mutate( + &app, + DynamicCapsMutateRequest::Share(request), + controller_local_only(&headers), + ) + .await? + { + DynamicCapsMutateResponse::Share(response) => Ok(Json(response)), + _ => unreachable!("share should return share response"), + } +} + +async fn control_dynamic_inspect_ref_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + match execute_site_controller_dynamic_caps_inspect( + &app, + DynamicCapsInspectRequest::InspectRef(request), + controller_local_only(&headers), + ) + .await? + { + DynamicCapsInspectResponse::InspectRef(response) => Ok(Json(response)), + _ => unreachable!("inspect_ref should return inspect response"), + } +} + +async fn control_dynamic_revoke_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + match execute_site_controller_dynamic_caps_mutate( + &app, + DynamicCapsMutateRequest::Revoke(request), + controller_local_only(&headers), + ) + .await? + { + DynamicCapsMutateResponse::Revoke(response) => Ok(Json(response)), + _ => unreachable!("revoke should return revoke response"), + } +} + +async fn control_dynamic_resolve_origin_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> +{ + authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + match execute_site_controller_dynamic_caps_inspect( + &app, + DynamicCapsInspectRequest::ResolveOrigin(request), + controller_local_only(&headers), + ) + .await? + { + DynamicCapsInspectResponse::ResolveOrigin(response) => Ok(Json(response)), + _ => unreachable!("resolve_origin should return origin resolution"), + } +} + +async fn site_prepare_child_route( + State(app): State, + headers: HeaderMap, + AxumPath(child_id): AxumPath, + Json(request): Json, +) -> std::result::Result { + authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + app.control + .runtime + .prepare_child(&app.control.controller_plan, child_id, request.site_plan) + .await + .map_err(|err| { + ProtocolApiError::from(protocol_error( + ProtocolErrorCode::PrepareFailed, + &err.to_string(), + )) + })?; + Ok(StatusCode::NO_CONTENT) +} + +async fn site_publish_child_route( + State(app): State, + headers: HeaderMap, + AxumPath(child_id): AxumPath, + Json(request): Json, +) -> std::result::Result { + authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + app.control + .runtime + .publish_child(&app.control.controller_plan, child_id, request.site_plan) + .await + .map_err(|err| { + ProtocolApiError::from(protocol_error( + ProtocolErrorCode::PublishFailed, + &err.to_string(), + )) + })?; + Ok(StatusCode::NO_CONTENT) +} + +async fn site_rollback_child_route( + State(app): State, + headers: HeaderMap, + AxumPath(child_id): AxumPath, +) -> std::result::Result { + authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + app.control + .runtime + .rollback_child(&app.control.controller_plan, child_id) + .await + .map_err(|err| { + ProtocolApiError::from(protocol_error( + ProtocolErrorCode::PublishFailed, + &err.to_string(), + )) + })?; + Ok(StatusCode::NO_CONTENT) +} + +async fn site_destroy_child_route( + State(app): State, + headers: HeaderMap, + AxumPath(child_id): AxumPath, + Json(request): Json, +) -> std::result::Result { + authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + app.control + .runtime + .destroy_child( + &app.control.controller_plan, + child_id, + request.desired_site_plan, + ) + .await + .map_err(|err| { + ProtocolApiError::from(protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &err.to_string(), + )) + })?; + Ok(StatusCode::NO_CONTENT) +} diff --git a/cli/src/framework_component/state.rs b/runtime/site-controller/src/state.rs similarity index 68% rename from cli/src/framework_component/state.rs rename to runtime/site-controller/src/state.rs index bb7f044c..1b07ce71 100644 --- a/cli/src/framework_component/state.rs +++ b/runtime/site-controller/src/state.rs @@ -2,12 +2,10 @@ use super::{http::*, orchestration::*, planner::*, *}; pub(super) const CONTROL_STATE_SCHEMA: &str = "amber.framework_component.control_state"; pub(super) const CONTROL_STATE_VERSION: u32 = 1; -pub(super) const CONTROL_SERVICE_PLAN_SCHEMA: &str = - "amber.framework_component.control_service_plan"; -pub(super) const CONTROL_SERVICE_PLAN_VERSION: u32 = 1; -pub(super) const CCS_PLAN_SCHEMA: &str = "amber.framework_component.ccs_plan"; -pub(super) const CCS_PLAN_VERSION: u32 = 1; -pub(super) const CONTROL_SERVICE_PATH: &str = "/v1/control-state"; +pub(super) const SITE_CONTROLLER_PLAN_SCHEMA: &str = + "amber.framework_component.site_controller_plan"; +pub(super) const SITE_CONTROLLER_PLAN_VERSION: u32 = 1; +pub(super) const SITE_CONTROLLER_STATE_PATH: &str = "/v1/controller/state"; pub(super) const FRAMEWORK_ROUTE_ID_HEADER: &str = "x-amber-route-id"; pub(super) const FRAMEWORK_PEER_ID_HEADER: &str = "x-amber-peer-id"; pub(super) const FRAMEWORK_AUTH_HEADER: &str = "x-amber-framework-auth"; @@ -86,52 +84,52 @@ pub(crate) struct PendingDestroyRecord { } #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct DynamicSitePlanRecord { - pub(crate) site_id: String, - pub(crate) kind: SiteKind, - pub(crate) router_identity_id: String, +pub struct DynamicSitePlanRecord { + pub site_id: String, + pub kind: SiteKind, + pub router_identity_id: String, #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub(crate) component_ids: Vec, + pub component_ids: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub(crate) assigned_components: Vec, + pub assigned_components: Vec, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub(crate) artifact_files: BTreeMap, + pub artifact_files: BTreeMap, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub(crate) desired_artifact_files: BTreeMap, + pub desired_artifact_files: BTreeMap, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub(crate) proxy_exports: BTreeMap, + pub proxy_exports: BTreeMap, #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub(crate) routed_inputs: Vec, + pub routed_inputs: Vec, } #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct DynamicProxyExportRecord { - pub(crate) component_id: usize, - pub(crate) component: String, - pub(crate) provide: String, - pub(crate) protocol: String, - pub(crate) capability_kind: String, +pub struct DynamicProxyExportRecord { + pub component_id: usize, + pub component: String, + pub provide: String, + pub protocol: String, + pub capability_kind: String, #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) capability_profile: Option, - pub(crate) target_port: u16, + pub capability_profile: Option, + pub target_port: u16, } #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct DynamicInputRouteRecord { - pub(crate) component: String, - pub(crate) slot: String, - pub(crate) provider_component: String, - pub(crate) protocol: String, - pub(crate) capability_kind: String, +pub struct DynamicInputRouteRecord { + pub component: String, + pub slot: String, + pub provider_component: String, + pub protocol: String, + pub capability_kind: String, #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) capability_profile: Option, + pub capability_profile: Option, #[serde(flatten)] - pub(crate) target: DynamicInputRouteTarget, + pub target: DynamicInputRouteTarget, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "route_kind", rename_all = "snake_case")] -pub(crate) enum DynamicInputRouteTarget { +pub enum DynamicInputRouteTarget { ComponentProvide { provide: String }, } @@ -211,7 +209,7 @@ pub(crate) struct ControlJournalEntry { } #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct FrameworkControlState { +pub struct FrameworkControlState { pub(crate) schema: String, pub(crate) version: u32, pub(crate) run_id: String, @@ -225,6 +223,8 @@ pub(crate) struct FrameworkControlState { pub(crate) next_child_id: u64, #[serde(default)] pub(crate) next_tx_id: u64, + #[serde(default = "default_framework_id_stride")] + pub(crate) id_stride: u64, #[serde(default)] pub(crate) next_component_id: usize, #[serde(default)] @@ -247,33 +247,174 @@ pub(crate) struct FrameworkControlState { pub(crate) pending_destroys: Vec, } +fn default_framework_id_stride() -> u64 { + 1 +} + #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct FrameworkControlStateServicePlan { - pub(crate) schema: String, - pub(crate) version: u32, - pub(crate) listen_addr: SocketAddr, - pub(crate) state_path: String, - pub(crate) run_root: String, - pub(crate) state_root: String, - pub(crate) mesh_scope: String, - pub(crate) auth_token: String, +pub struct SiteControllerPeerPlan { + pub site_id: String, + pub kind: SiteKind, + pub authority_url: String, } #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct FrameworkCcsPlan { - pub(crate) schema: String, - pub(crate) version: u32, - pub(crate) site_id: String, - pub(crate) site_state_root: String, - pub(crate) listen_addr: SocketAddr, - pub(crate) control_state_url: String, - pub(crate) router_auth_token: String, - pub(crate) control_state_auth_token: String, +pub struct SiteControllerPlan { + pub schema: String, + pub version: u32, + pub run_id: String, + pub mesh_scope: String, + pub site_id: String, + pub kind: SiteKind, + pub listen_addr: SocketAddr, + pub authority_url: String, + pub router_identity_id: String, + pub state_path: String, + pub run_root: String, + pub state_root: String, + pub site_state_root: String, + pub artifact_dir: String, + pub auth_token: String, + pub dynamic_caps_token_verify_key_b64: String, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub peer_controllers: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage_root: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub runtime_root: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub router_mesh_port: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub compose_project: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub kubernetes_namespace: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observability_endpoint: Option, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub launch_env: BTreeMap, } +#[cfg(test)] pub(crate) fn build_control_state( run_id: &str, run_plan: &RunPlan, +) -> Result { + build_control_state_with_signing_seed( + run_id, + run_plan, + &amber_mesh::dynamic_caps::signing_seed_b64( + &amber_mesh::dynamic_caps::signing_key_from_seed( + amber_mesh::dynamic_caps::generate_dynamic_capability_signing_seed(), + ), + ), + ) +} + +pub fn build_site_controller_state( + run_id: &str, + run_plan: &RunPlan, + site_id: &str, + site_index: usize, + site_count: usize, + dynamic_capability_signing_seed_b64: &str, +) -> Result { + let mut state = build_control_state_with_signing_seed( + run_id, + run_plan, + dynamic_capability_signing_seed_b64, + )?; + localize_framework_control_state(&mut state, site_id)?; + let site_offset = site_index as u64; + let id_stride = site_count.max(1) as u64; + state.next_child_id = localized_last_allocated_counter( + site_offset, + id_stride, + state + .live_children + .iter() + .map(|child| child.child_id) + .chain( + state + .pending_creates + .iter() + .map(|record| record.child.child_id), + ) + .chain( + state + .pending_destroys + .iter() + .map(|record| record.child.child_id), + ), + ); + state.next_tx_id = localized_last_allocated_counter( + site_offset, + id_stride, + state + .pending_creates + .iter() + .map(|record| record.tx_id) + .chain(state.pending_destroys.iter().map(|record| record.tx_id)) + .chain(state.journal.iter().map(|entry| entry.tx_id)), + ); + state.next_dynamic_capability_grant_id = localized_next_available_counter( + site_offset, + id_stride, + state + .dynamic_capability_grants + .keys() + .filter_map(|grant_id| dynamic_capability_grant_counter(grant_id)) + .chain( + state + .dynamic_capability_journal + .iter() + .filter_map(|entry| entry.grant_id.as_deref()) + .filter_map(dynamic_capability_grant_counter), + ), + ); + state.id_stride = id_stride; + Ok(state) +} + +fn localized_last_allocated_counter( + site_offset: u64, + id_stride: u64, + used_ids: impl Iterator, +) -> u64 { + let max_used_id = used_ids.max(); + match max_used_id { + Some(max_used_id) if max_used_id > site_offset => { + site_offset + ((max_used_id - site_offset) / id_stride.max(1)) * id_stride.max(1) + } + _ => site_offset, + } +} + +fn localized_next_available_counter( + site_offset: u64, + id_stride: u64, + used_ids: impl Iterator, +) -> u64 { + let max_used_id = used_ids.max(); + match max_used_id { + Some(max_used_id) if max_used_id >= site_offset => { + site_offset + (((max_used_id - site_offset) / id_stride.max(1)) + 1) * id_stride.max(1) + } + _ => site_offset, + } +} + +fn dynamic_capability_grant_counter(grant_id: &str) -> Option { + grant_id + .strip_prefix(dynamic_caps::DYNAMIC_CAPABILITY_GRANT_ID_PREFIX) + .and_then(|suffix| u64::from_str_radix(suffix, 16).ok()) +} + +fn build_control_state_with_signing_seed( + run_id: &str, + run_plan: &RunPlan, + dynamic_capability_signing_seed_b64: &str, ) -> Result { let scenario = Scenario::try_from(run_plan.base_scenario.clone()) .into_diagnostic() @@ -304,14 +445,11 @@ pub(crate) fn build_control_state( generation: 0, next_child_id: 0, next_tx_id: 0, + id_stride: default_framework_id_stride(), next_component_id, capability_instances: BTreeMap::new(), journal: Vec::new(), - dynamic_capability_signing_seed_b64: amber_mesh::dynamic_caps::signing_seed_b64( - &amber_mesh::dynamic_caps::signing_key_from_seed( - amber_mesh::dynamic_caps::generate_dynamic_capability_signing_seed(), - ), - ), + dynamic_capability_signing_seed_b64: dynamic_capability_signing_seed_b64.to_string(), next_dynamic_capability_grant_id: 0, dynamic_capability_grants: BTreeMap::new(), dynamic_capability_journal: Vec::new(), @@ -331,6 +469,134 @@ pub(crate) fn build_control_state( Ok(state) } +pub(crate) fn localize_framework_control_state( + state: &mut FrameworkControlState, + site_id: &str, +) -> Result<()> { + let local_child_ids = state + .live_children + .iter() + .chain(state.pending_creates.iter().map(|record| &record.child)) + .chain(state.pending_destroys.iter().map(|record| &record.child)) + .filter_map(|child| { + child_authority_site_id(state, child) + .ok() + .filter(|authority_site_id| authority_site_id == site_id) + .map(|_| child.child_id) + }) + .collect::>(); + state + .live_children + .retain(|child| local_child_ids.contains(&child.child_id)); + state + .pending_creates + .retain(|record| local_child_ids.contains(&record.child.child_id)); + state + .pending_destroys + .retain(|record| local_child_ids.contains(&record.child.child_id)); + state + .journal + .retain(|entry| local_child_ids.contains(&entry.child_id)); + + let roots = dynamic_caps::derive_root_authorities(state) + .map_err(|err| miette::miette!(err.message.clone()))?; + let local_grant_ids = state + .dynamic_capability_grants + .iter() + .filter_map(|(grant_id, grant)| { + roots + .get(&dynamic_caps::root_authority_key( + &grant.root_authority_selector, + )) + .and_then(|root| component_site_id(state, &root.holder_component_id).ok()) + .filter(|root_site_id| root_site_id == site_id) + .map(|_| grant_id.clone()) + }) + .collect::>(); + state + .dynamic_capability_grants + .retain(|grant_id, _| local_grant_ids.contains(grant_id)); + state.dynamic_capability_journal.retain(|entry| { + entry + .grant_id + .as_ref() + .is_none_or(|grant_id| local_grant_ids.contains(grant_id)) + }); + Ok(()) +} + +fn child_authority_site_id( + state: &FrameworkControlState, + child: &LiveChildRecord, +) -> std::result::Result { + let authority_moniker = decode_live_scenario(state)? + .component(ComponentId(child.authority_realm_id)) + .moniker + .to_string(); + site_id_for_moniker( + state, + &authority_moniker, + &format!("authority realm `{authority_moniker}`"), + ) +} + +fn component_site_id( + state: &FrameworkControlState, + logical_component_id: &str, +) -> std::result::Result { + let moniker = dynamic_caps::moniker_from_logical_component_id(logical_component_id)?; + let moniker = moniker.to_string(); + site_id_for_moniker( + state, + &moniker, + &format!("component `{logical_component_id}`"), + ) +} + +fn site_id_for_moniker( + state: &FrameworkControlState, + moniker: &str, + subject: &str, +) -> std::result::Result { + let assignments = live_assignment_map(state); + if let Some(site_id) = assignments.get(moniker) { + return Ok(site_id.clone()); + } + + let descendant_sites = assignments + .iter() + .filter(|(assigned_moniker, _)| moniker_contains(assigned_moniker, moniker)) + .map(|(_, site_id)| site_id.clone()) + .collect::>(); + if descendant_sites.len() == 1 { + return Ok(descendant_sites + .into_iter() + .next() + .expect("single descendant site should be present")); + } + + Err(protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &if descendant_sites.is_empty() { + format!("{subject} is missing a site assignment") + } else { + format!( + "{subject} is missing a site assignment and spans multiple sites: {}", + descendant_sites.into_iter().collect::>().join(", ") + ) + }, + )) +} + +fn moniker_contains(candidate: &str, realm: &str) -> bool { + if realm == "/" { + return true; + } + candidate + .strip_prefix(realm) + .is_some_and(|suffix| suffix.starts_with('/')) +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub(super) struct FrameworkChildSnapshotRecord { pub(super) child: FrameworkChildSnapshotState, @@ -776,7 +1042,7 @@ fn remap_output_handle_for_snapshot( }) } -pub(crate) fn generate_framework_auth_token(mesh_scope: &str, purpose: &str) -> String { +pub fn generate_framework_auth_token(mesh_scope: &str, purpose: &str) -> String { base64::engine::general_purpose::STANDARD.encode( MeshIdentity::generate( format!("/framework/{purpose}"), @@ -786,23 +1052,16 @@ pub(crate) fn generate_framework_auth_token(mesh_scope: &str, purpose: &str) -> ) } -pub(crate) fn control_state_service_url(listen_addr: SocketAddr) -> String { - format!("http://{listen_addr}") -} - -pub(crate) fn ccs_listen_addr_for_site(kind: SiteKind, port: u16) -> SocketAddr { - host_service_bind_addr_for_consumer(kind, port) -} - -pub(crate) fn ccs_url_for_site(kind: SiteKind, port: u16) -> String { - let host = match kind { - SiteKind::Direct | SiteKind::Vm => Ipv4Addr::LOCALHOST.to_string(), - SiteKind::Compose | SiteKind::Kubernetes => host_service_host_for_consumer(kind), +pub fn authority_url_for_listen_addr(listen_addr: SocketAddr) -> String { + let dial_addr = if listen_addr.ip().is_unspecified() { + SocketAddr::from(([127, 0, 0, 1], listen_addr.port())) + } else { + listen_addr }; - format!("http://{host}:{port}") + format!("http://{dial_addr}") } -pub(crate) fn write_control_state(path: &Path, state: &FrameworkControlState) -> Result<()> { +pub fn write_control_state(path: &Path, state: &FrameworkControlState) -> Result<()> { write_json(path, state) } @@ -937,48 +1196,122 @@ pub(super) fn child_destroy_tx_id( }) } -pub(crate) fn write_control_state_service_plan( +#[allow(clippy::too_many_arguments)] +pub fn write_site_controller_plan( path: &Path, + run_id: &str, + mesh_scope: &str, + site_id: &str, + kind: SiteKind, listen_addr: SocketAddr, + authority_url: &str, + router_identity_id: &str, state_path: &Path, run_root: &Path, state_root: &Path, - mesh_scope: &str, + site_state_root: &Path, + artifact_dir: &Path, auth_token: &str, -) -> Result { - let plan = FrameworkControlStateServicePlan { - schema: CONTROL_SERVICE_PLAN_SCHEMA.to_string(), - version: CONTROL_SERVICE_PLAN_VERSION, + dynamic_caps_token_verify_key_b64: &str, + peer_controllers: BTreeMap, + storage_root: Option<&str>, + runtime_root: Option<&str>, + router_mesh_port: Option, + compose_project: Option<&str>, + kubernetes_namespace: Option<&str>, + context: Option<&str>, + observability_endpoint: Option<&str>, + launch_env: &BTreeMap, +) -> Result { + let plan = SiteControllerPlan { + schema: SITE_CONTROLLER_PLAN_SCHEMA.to_string(), + version: SITE_CONTROLLER_PLAN_VERSION, + run_id: run_id.to_string(), + mesh_scope: mesh_scope.to_string(), + site_id: site_id.to_string(), + kind, listen_addr, + authority_url: authority_url.to_string(), + router_identity_id: router_identity_id.to_string(), state_path: state_path.display().to_string(), run_root: run_root.display().to_string(), state_root: state_root.display().to_string(), - mesh_scope: mesh_scope.to_string(), + site_state_root: site_state_root.display().to_string(), + artifact_dir: artifact_dir.display().to_string(), auth_token: auth_token.to_string(), + dynamic_caps_token_verify_key_b64: dynamic_caps_token_verify_key_b64.to_string(), + peer_controllers, + storage_root: storage_root.map(str::to_string), + runtime_root: runtime_root.map(str::to_string), + router_mesh_port, + compose_project: compose_project.map(str::to_string), + kubernetes_namespace: kubernetes_namespace.map(str::to_string), + context: context.map(str::to_string), + observability_endpoint: observability_endpoint.map(str::to_string), + launch_env: launch_env.clone(), }; write_json(path, &plan)?; Ok(plan) } -pub(crate) fn write_framework_ccs_plan( - path: &Path, - site_id: &str, - site_state_root: &Path, - listen_addr: SocketAddr, - control_state_url: &str, - router_auth_token: &str, - control_state_auth_token: &str, -) -> Result { - let plan = FrameworkCcsPlan { - schema: CCS_PLAN_SCHEMA.to_string(), - version: CCS_PLAN_VERSION, - site_id: site_id.to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr, - control_state_url: control_state_url.to_string(), - router_auth_token: router_auth_token.to_string(), - control_state_auth_token: control_state_auth_token.to_string(), - }; - write_json(path, &plan)?; - Ok(plan) +pub(crate) fn site_id_for_authority_realm( + state: &FrameworkControlState, + authority_realm_id: usize, +) -> std::result::Result { + let authority_moniker = decode_live_scenario(state)? + .component(ComponentId(authority_realm_id)) + .moniker + .to_string(); + site_id_for_moniker( + state, + authority_moniker.as_str(), + &format!("authority realm `{authority_moniker}`"), + ) +} + +pub(crate) fn site_id_for_dynamic_grant( + state: &FrameworkControlState, + grant_id: &str, +) -> std::result::Result { + let grant = state + .dynamic_capability_grants + .get(grant_id) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::UnknownSource, + &format!("dynamic grant `{grant_id}` is not live"), + ) + })?; + site_id_for_logical_component(state, &grant.holder_component_id).map_err(|_| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "dynamic grant `{grant_id}` holder `{}` is missing a live site assignment", + grant.holder_component_id + ), + ) + }) +} + +pub(crate) fn site_id_for_logical_component( + state: &FrameworkControlState, + logical_component_id: &str, +) -> std::result::Result { + component_site_id(state, logical_component_id) +} + +pub(crate) fn site_id_for_root_authority_selector( + state: &FrameworkControlState, + selector: &RootAuthoritySelectorIr, +) -> std::result::Result { + let roots = dynamic_caps::derive_root_authorities(state)?; + let root = roots + .get(&dynamic_caps::root_authority_key(selector)) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::UnknownSource, + "dynamic capability root authority is not live", + ) + })?; + site_id_for_logical_component(state, &root.holder_component_id) } diff --git a/cli/src/framework_component/tests.rs b/runtime/site-controller/src/tests.rs similarity index 84% rename from cli/src/framework_component/tests.rs rename to runtime/site-controller/src/tests.rs index 7e6a96d7..bf6c4823 100644 --- a/cli/src/framework_component/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -10,6 +10,7 @@ use amber_mesh::{ HeldEntryKind, HeldEntryState, RootAuthoritySelectorIr, }, }; +use axum::{Router, extract::State, http::HeaderMap, routing::post}; use reqwest::{Client, StatusCode}; use serde::{Serialize, de::DeserializeOwned}; use serde_json::{Value, json}; @@ -18,6 +19,356 @@ use url::Url; use super::{api::*, http::*, orchestration::*, planner::*, state::*, *}; +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +struct DirectRuntimeState { + #[serde(default)] + slot_ports_by_component: BTreeMap>, + #[serde(default)] + slot_route_ports_by_component: BTreeMap>>, + #[serde(default)] + dynamic_caps_port_by_component: BTreeMap, + #[serde(default)] + component_mesh_port_by_id: BTreeMap, + #[serde(default)] + router_mesh_port: Option, +} + +fn direct_runtime_state_path(plan_root: &Path) -> PathBuf { + plan_root.join(".amber").join("direct-runtime.json") +} + +#[derive(Clone, Default)] +struct TestSiteControllerRuntime; + +impl SiteControllerRuntime for TestSiteControllerRuntime { + fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn resolve_link_external_url<'a>( + &'a self, + _provider: &'a LaunchedSite, + _provider_output_dir: &'a Path, + link: &'a amber_compiler::run_plan::RunLink, + _consumer_kind: SiteKind, + _run_root: &'a Path, + ) -> SiteControllerRuntimeFuture<'a, String> { + Box::pin(async move { + Ok(match link.protocol { + amber_manifest::NetworkProtocol::Tcp => "tcp://127.0.0.1:1".to_string(), + _ => "http://127.0.0.1:1".to_string(), + }) + }) + } + + fn prepare_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _child_id: u64, + _site_plan: DynamicSitePlanRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn publish_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _child_id: u64, + _site_plan: DynamicSitePlanRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn rollback_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _child_id: u64, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn destroy_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _child_id: u64, + _desired_site_plan: Option, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn collect_live_component_runtime_metadata( + &self, + plan: &SiteControllerRuntimePlan, + ) -> miette::Result> { + if plan.kind != SiteKind::Direct { + return Err(miette::miette!( + "test runtime only implements direct site runtime metadata" + )); + } + let artifact_dir = PathBuf::from(&plan.artifact_dir); + let runtime_root = PathBuf::from( + plan.runtime_root + .as_deref() + .ok_or_else(|| miette::miette!("direct test site is missing its runtime root"))?, + ); + let direct_plan: serde_json::Value = + read_json(&artifact_dir.join("direct-plan.json"), "direct plan")?; + let runtime_state: DirectRuntimeState = read_json( + &direct_runtime_state_path(&artifact_dir), + "direct runtime state", + )?; + let mut components = BTreeMap::new(); + for component in direct_plan["components"].as_array().into_iter().flatten() { + let Some(component_id) = component.get("id").and_then(serde_json::Value::as_u64) else { + continue; + }; + let Some(moniker) = component.get("moniker").and_then(serde_json::Value::as_str) else { + continue; + }; + let Some(port) = runtime_state + .component_mesh_port_by_id + .get(&(component_id as usize)) + .copied() + else { + continue; + }; + let Some(mesh_config_path) = component + .get("sidecar") + .and_then(|value| value.get("mesh_config_path")) + .and_then(serde_json::Value::as_str) + else { + continue; + }; + let mesh_config = read_json( + &runtime_root.join(mesh_config_path), + "component mesh config", + )?; + components.insert( + moniker.to_string(), + LiveComponentRuntimeMetadata { + moniker: moniker.to_string(), + host_mesh_addr: format!("127.0.0.1:{port}"), + mesh_config, + }, + ); + } + Ok(components) + } + + fn load_live_site_router_mesh_config( + &self, + plan: &SiteControllerRuntimePlan, + ) -> miette::Result { + if plan.kind != SiteKind::Direct { + return Err(miette::miette!( + "test runtime only implements direct site router metadata" + )); + } + let artifact_dir = PathBuf::from(&plan.artifact_dir); + let runtime_root = PathBuf::from( + plan.runtime_root + .as_deref() + .ok_or_else(|| miette::miette!("direct test site is missing its runtime root"))?, + ); + let direct_plan: serde_json::Value = + read_json(&artifact_dir.join("direct-plan.json"), "direct plan")?; + let mesh_config_path = direct_plan["router"]["mesh_config_path"] + .as_str() + .ok_or_else(|| { + miette::miette!("direct test site is missing router mesh config path") + })?; + read_json(&runtime_root.join(mesh_config_path), "router mesh config") + } + + fn router_mesh_addr_for_consumer( + &self, + _provider_kind: SiteKind, + _consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> miette::Result { + Ok(router_mesh_addr.to_string()) + } + + fn update_desired_overlay_for_consumer( + &self, + _site_state_root: &Path, + _overlay_id: &str, + _overlay: DesiredExternalSlotOverlay, + ) -> miette::Result<()> { + Ok(()) + } + + fn update_desired_overlay_for_provider( + &self, + _site_state_root: &Path, + _overlay_id: &str, + _overlay: DesiredExportPeerOverlay, + ) -> miette::Result<()> { + Ok(()) + } + + fn clear_desired_overlay_for_consumer( + &self, + _site_state_root: &Path, + _overlay_id: &str, + ) -> miette::Result<()> { + Ok(()) + } + + fn clear_desired_overlay_for_provider( + &self, + _site_state_root: &Path, + _overlay_id: &str, + ) -> miette::Result<()> { + Ok(()) + } +} + +fn test_runtime() -> SharedSiteControllerRuntime { + Arc::new(TestSiteControllerRuntime) +} + +type DestroyCalls = Arc)>>>; + +#[derive(Clone, Default)] +struct FailingPublishRuntime { + destroy_calls: DestroyCalls, +} + +impl SiteControllerRuntime for FailingPublishRuntime { + fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn resolve_link_external_url<'a>( + &'a self, + _provider: &'a LaunchedSite, + _provider_output_dir: &'a Path, + _link: &'a amber_compiler::run_plan::RunLink, + _consumer_kind: SiteKind, + _run_root: &'a Path, + ) -> SiteControllerRuntimeFuture<'a, String> { + Box::pin(async { Ok("http://127.0.0.1:1".to_string()) }) + } + + fn prepare_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _child_id: u64, + _site_plan: DynamicSitePlanRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn publish_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _child_id: u64, + _site_plan: DynamicSitePlanRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Err(miette::miette!("publish exploded")) }) + } + + fn rollback_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _child_id: u64, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn destroy_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + child_id: u64, + desired_site_plan: Option, + ) -> SiteControllerRuntimeFuture<'a, ()> { + let destroy_calls = self.destroy_calls.clone(); + Box::pin(async move { + destroy_calls + .lock() + .expect("destroy call log mutex should lock") + .push((child_id, desired_site_plan)); + Ok(()) + }) + } + + fn collect_live_component_runtime_metadata( + &self, + _plan: &SiteControllerRuntimePlan, + ) -> miette::Result> { + Ok(BTreeMap::new()) + } + + fn load_live_site_router_mesh_config( + &self, + _plan: &SiteControllerRuntimePlan, + ) -> miette::Result { + Err(miette::miette!( + "failing publish runtime should not load live site router mesh config" + )) + } + + fn router_mesh_addr_for_consumer( + &self, + _provider_kind: SiteKind, + _consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> miette::Result { + Ok(router_mesh_addr.to_string()) + } + + fn update_desired_overlay_for_consumer( + &self, + _site_state_root: &Path, + _overlay_id: &str, + _overlay: DesiredExternalSlotOverlay, + ) -> miette::Result<()> { + Ok(()) + } + + fn update_desired_overlay_for_provider( + &self, + _site_state_root: &Path, + _overlay_id: &str, + _overlay: DesiredExportPeerOverlay, + ) -> miette::Result<()> { + Ok(()) + } + + fn clear_desired_overlay_for_consumer( + &self, + _site_state_root: &Path, + _overlay_id: &str, + ) -> miette::Result<()> { + Ok(()) + } + + fn clear_desired_overlay_for_provider( + &self, + _site_state_root: &Path, + _overlay_id: &str, + ) -> miette::Result<()> { + Ok(()) + } +} + +fn with_runtime(app: &ControlStateApp, runtime: SharedSiteControllerRuntime) -> ControlStateApp { + ControlStateApp { + control_state: app.control_state.clone(), + client: app.client.clone(), + state_path: app.state_path.clone(), + run_root: app.run_root.clone(), + state_root: app.state_root.clone(), + mesh_scope: app.mesh_scope.clone(), + control_state_auth_token: app.control_state_auth_token.clone(), + controller_plan: app.controller_plan.clone(), + peer_controllers: app.peer_controllers.clone(), + authority_locks: app.authority_locks.clone(), + runtime, + } +} + fn write_file(path: &Path, contents: &str) { fs::write(path, contents).expect("test fixture should write"); } @@ -432,7 +783,7 @@ async fn compile_dynamic_caps_binding_state() -> FrameworkControlState { path: Some("direct_local".to_string()), ..PlacementDefaults::default() }, - components: BTreeMap::new(), + components: BTreeMap::from([("/".to_string(), "direct_local".to_string())]), dynamic_capabilities: None, framework_children: None, }; @@ -1030,7 +1381,7 @@ async fn same_site_dynamic_child_output_bindings_reuse_provider_component_routes write_control_state(&state_path, &state).expect("state should write"); let root_authority = state.base_scenario.root; let app = test_control_state_app(&dir, state, state_path); - let actuators = install_success_site_actuator(&app).await; + let (app, controllers) = install_success_site_controllers(&app).await; execute_create_child( &app, @@ -1088,8 +1439,8 @@ async fn same_site_dynamic_child_output_bindings_reuse_provider_component_routes ); } - for actuator in actuators { - actuator.abort(); + for controller in controllers { + controller.abort(); } } @@ -1188,7 +1539,7 @@ async fn same_site_static_child_export_bindings_reuse_provider_component_routes( write_control_state(&state_path, &state).expect("state should write"); let root_authority = state.base_scenario.root; let app = test_control_state_app(&dir, state, state_path); - let actuators = install_success_site_actuator(&app).await; + let (app, controllers) = install_success_site_controllers(&app).await; execute_create_child( &app, @@ -1232,47 +1583,61 @@ async fn same_site_static_child_export_bindings_reuse_provider_component_routes( ); } - for actuator in actuators { - actuator.abort(); + for controller in controllers { + controller.abort(); } } #[test] -fn framework_ccs_addressing_matches_site_runtime_topology() { - assert_eq!( - ccs_listen_addr_for_site(SiteKind::Direct, 41000), - SocketAddr::from(([127, 0, 0, 1], 41000)) - ); +fn controller_authority_url_normalizes_unspecified_bind_addresses() { assert_eq!( - ccs_url_for_site(SiteKind::Direct, 41000), + authority_url_for_listen_addr(SocketAddr::from(([127, 0, 0, 1], 41000))), "http://127.0.0.1:41000" ); assert_eq!( - ccs_listen_addr_for_site(SiteKind::Vm, 42000), - SocketAddr::from(([127, 0, 0, 1], 42000)) - ); - assert_eq!( - ccs_url_for_site(SiteKind::Vm, 42000), + authority_url_for_listen_addr(SocketAddr::from(([0, 0, 0, 0], 42000))), "http://127.0.0.1:42000" ); - assert_eq!( - ccs_listen_addr_for_site(SiteKind::Compose, 43000), - SocketAddr::from(([0, 0, 0, 0], 43000)) - ); - assert_eq!( - ccs_url_for_site(SiteKind::Compose, 43000), - "http://host.docker.internal:43000" - ); - assert_eq!( - ccs_listen_addr_for_site(SiteKind::Kubernetes, 44000), - SocketAddr::from(([0, 0, 0, 0], 44000)) +} + +#[tokio::test] +async fn dynamic_grant_routes_to_holder_site_not_offered_site_order() { + let mut state = compile_dynamic_caps_binding_state().await; + state.placement.offered_sites.insert( + "compose_local".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + }, ); + let share = super::dynamic_caps::share_dynamic_capability( + &mut state, + "components./provider", + &super::dynamic_caps::source_key_from_control_request( + &dynamic_caps::DynamicCapabilityControlSourceRequest::RootAuthority { + root_authority_selector: RootAuthoritySelectorIr::SelfProvide { + component_id: "components./provider".to_string(), + provide_name: "http".to_string(), + }, + }, + ), + "components./alice", + None, + &json!({}), + ) + .expect("share should succeed"); + let grant_id = match share { + super::dynamic_caps::DynamicCapabilityShareOutcome::Created { grant_id, .. } + | super::dynamic_caps::DynamicCapabilityShareOutcome::Deduplicated { grant_id, .. } => { + grant_id + } + super::dynamic_caps::DynamicCapabilityShareOutcome::Noop { reason } => { + panic!("share should not noop: {reason}"); + } + }; assert_eq!( - ccs_url_for_site(SiteKind::Kubernetes, 44000), - format!( - "http://{}:44000", - host_service_host_for_consumer(SiteKind::Kubernetes) - ) + site_id_for_dynamic_grant(&state, &grant_id).expect("grant site should resolve"), + "direct_local", ); } @@ -1288,7 +1653,25 @@ async fn compile_empty_control_state() -> (TempDir, FrameworkControlState, PathB } "#, ); - let state = compile_control_state(&root_path).await; + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + )]), + defaults: PlacementDefaults { + path: Some("direct_local".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::new(), + dynamic_capabilities: None, + framework_children: None, + }; + let state = compile_control_state_with_placement(&root_path, Some(&placement)).await; let state_path = dir.path().join("control-state.json"); write_control_state(&state_path, &state).expect("state should write"); (dir, state, state_path) @@ -1331,7 +1714,25 @@ async fn compile_exact_template_control_state() -> (TempDir, FrameworkControlSta child = file_url(&child_path), ), ); - let state = compile_control_state(&root_path).await; + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + )]), + defaults: PlacementDefaults { + path: Some("direct_local".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([("/".to_string(), "direct_local".to_string())]), + dynamic_capabilities: None, + framework_children: None, + }; + let state = compile_control_state_with_placement(&root_path, Some(&placement)).await; let state_path = dir.path().join("control-state.json"); write_control_state(&state_path, &state).expect("state should write"); (dir, state, state_path) @@ -1365,8 +1766,8 @@ async fn compile_framework_binding_control_state() -> ( { manifest_version: "0.3.0", program: { - path: "/bin/echo", - args: ["worker"], + path: "/usr/bin/env", + args: ["python3", "-m", "http.server", "8080"], network: { endpoints: [{ name: "out", port: 8080, protocol: "http" }] } }, provides: { out: { kind: "http", endpoint: "out" } }, @@ -1399,7 +1800,25 @@ async fn compile_framework_binding_control_state() -> ( ), ); - let state = compile_control_state(&root_path).await; + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + )]), + defaults: PlacementDefaults { + path: Some("direct_local".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([("/".to_string(), "direct_local".to_string())]), + dynamic_capabilities: None, + framework_children: None, + }; + let state = compile_control_state_with_placement(&root_path, Some(&placement)).await; let state_path = dir.path().join("control-state.json"); write_control_state(&state_path, &state).expect("state should write"); let record = state @@ -1451,16 +1870,55 @@ fn test_control_state_app( let state_root = dir.path().join("state"); fs::create_dir_all(&run_root).expect("run root should exist"); fs::create_dir_all(&state_root).expect("state root should exist"); + let site_id = state + .placement + .offered_sites + .keys() + .next() + .cloned() + .unwrap_or_else(|| "direct_local".to_string()); + let site_state_root = state_root.join(&site_id); + let artifact_dir = dir.path().join("artifact").join(&site_id); + let storage_root = dir.path().join("storage").join(&site_id); + let runtime_root = dir.path().join("runtime").join(&site_id); ControlStateApp { control_state: Arc::new(Mutex::new(state)), client: ReqwestClient::new(), - state_path, - run_root, - state_root, + state_path: state_path.clone(), + run_root: run_root.clone(), + state_root: state_root.clone(), mesh_scope: Arc::::from("test-mesh"), control_state_auth_token: Arc::::from("test-control-state-auth"), + controller_plan: Arc::new(SiteControllerPlan { + schema: "amber.framework_component.site_controller_plan".to_string(), + version: 1, + run_id: "test-run".to_string(), + mesh_scope: "test-mesh".to_string(), + site_id: site_id.clone(), + kind: SiteKind::Direct, + listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), + authority_url: "http://127.0.0.1:0".to_string(), + router_identity_id: format!("/site/{site_id}/router"), + state_path: state_path.display().to_string(), + run_root: run_root.display().to_string(), + state_root: state_root.display().to_string(), + site_state_root: site_state_root.display().to_string(), + artifact_dir: artifact_dir.display().to_string(), + auth_token: "test-control-state-auth".to_string(), + dynamic_caps_token_verify_key_b64: String::new(), + peer_controllers: BTreeMap::new(), + storage_root: Some(storage_root.display().to_string()), + runtime_root: Some(runtime_root.display().to_string()), + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: None, + context: None, + observability_endpoint: None, + launch_env: BTreeMap::new(), + }), + peer_controllers: Arc::new(BTreeMap::new()), authority_locks: Arc::new(Mutex::new(BTreeMap::new())), - bridge_proxies: Arc::new(Mutex::new(BTreeMap::new())), + runtime: test_runtime(), } } @@ -1487,6 +1945,13 @@ async fn spawn_test_router(router: Router) -> (String, tokio::task::JoinHandle<( let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) .await .expect("test listener"); + spawn_test_router_on(listener, router) +} + +fn spawn_test_router_on( + listener: tokio::net::TcpListener, + router: Router, +) -> (String, tokio::task::JoinHandle<()>) { let addr = listener.local_addr().expect("test listener addr"); let handle = tokio::spawn(async move { axum::serve(listener, router.into_make_service()) @@ -1604,6 +2069,121 @@ fn normalize_dynamic_share_ref(value: &mut Value) { } } +const TEST_SITE_STATE_SCHEMA: &str = "amber.run.site_state"; +const TEST_SITE_STATE_VERSION: u32 = 2; + +fn with_controller_endpoint( + app: &ControlStateApp, + authority_url: &str, + listen_addr: SocketAddr, +) -> ControlStateApp { + let mut controller_plan = app.controller_plan.as_ref().clone(); + controller_plan.authority_url = authority_url.to_string(); + controller_plan.listen_addr = listen_addr; + ControlStateApp { + control_state: app.control_state.clone(), + client: app.client.clone(), + state_path: app.state_path.clone(), + run_root: app.run_root.clone(), + state_root: app.state_root.clone(), + mesh_scope: app.mesh_scope.clone(), + control_state_auth_token: app.control_state_auth_token.clone(), + controller_plan: Arc::new(controller_plan), + peer_controllers: app.peer_controllers.clone(), + authority_locks: app.authority_locks.clone(), + runtime: app.runtime.clone(), + } +} + +async fn install_framework_site_controller_fixture( + app: &ControlStateApp, +) -> Vec> { + let plan = app.controller_plan.as_ref(); + let site_state_root = PathBuf::from(&plan.site_state_root); + let artifact_dir = PathBuf::from(&plan.artifact_dir); + let runtime_root = PathBuf::from( + plan.runtime_root + .as_deref() + .expect("test controller plan should include runtime root"), + ); + fs::create_dir_all(artifact_dir.join(".amber").join("control")) + .expect("artifact control dir should exist"); + fs::create_dir_all(&runtime_root).expect("runtime root should exist"); + fs::create_dir_all(&site_state_root).expect("site state root should exist"); + write_json(&site_controller_plan_path(&site_state_root), plan) + .expect("site controller plan should write"); + + let overlay_router = Router::new().route( + "/overlays/{overlay_id}", + axum::routing::put(|| async { StatusCode::NO_CONTENT }) + .delete(|| async { StatusCode::NO_CONTENT }), + ); + let (router_base_url, overlay_handle) = spawn_test_router(overlay_router).await; + let router_control = router_base_url + .strip_prefix("http://") + .expect("router control URL should be absolute HTTP") + .to_string(); + + write_json( + &site_state_path(&app.state_root, &plan.site_id), + &json!({ + "schema": TEST_SITE_STATE_SCHEMA, + "version": TEST_SITE_STATE_VERSION, + "run_id": plan.run_id, + "site_id": plan.site_id, + "kind": plan.kind, + "status": "running", + "artifact_dir": plan.artifact_dir, + "supervisor_pid": 1, + "router_control": router_control, + "router_mesh_addr": format!("127.0.0.1:{}", plan.router_mesh_port.expect("router mesh port")), + "router_identity_id": plan.router_identity_id, + "router_public_key_b64": base64::engine::general_purpose::STANDARD.encode([11; 32]), + "site_controller_pid": 1, + "site_controller_url": plan.authority_url, + }), + ) + .expect("manager state should write"); + write_json( + &artifact_dir.join("direct-plan.json"), + &json!({ + "version": "3", + "mesh_provision_plan": "mesh-provision-plan.json", + "startup_order": [], + "components": [], + "router": { + "identity_id": plan.router_identity_id, + "mesh_port": plan.router_mesh_port.expect("router mesh port"), + "control_port": 39011, + "control_socket_path": "router.sock", + "mesh_config_path": "router-mesh.json", + "mesh_identity_path": "router-identity.json", + }, + }), + ) + .expect("direct plan should write"); + write_json( + &direct_runtime_state_path(&artifact_dir), + &DirectRuntimeState { + router_mesh_port: plan.router_mesh_port, + ..Default::default() + }, + ) + .expect("direct runtime state should write"); + let mut router_mesh = test_live_site_router(Vec::new()); + router_mesh.identity.id = plan.router_identity_id.clone(); + router_mesh.mesh_listen = format!( + "127.0.0.1:{}", + plan.router_mesh_port.expect("router mesh port") + ) + .parse() + .expect("router mesh listen should parse"); + write_json(&runtime_root.join("router-mesh.json"), &router_mesh) + .expect("router mesh config should write"); + + vec![overlay_handle] +} + struct TestMcpClient { client: Client, endpoint: String, @@ -1614,8 +2194,15 @@ struct TestMcpClient { impl TestMcpClient { async fn connect(base_url: &str, client_name: &str, headers: Vec<(String, String)>) -> Self { + Self::connect_endpoint(&format!("{base_url}/mcp"), client_name, headers).await + } + + async fn connect_endpoint( + endpoint: &str, + client_name: &str, + headers: Vec<(String, String)>, + ) -> Self { let client = Client::new(); - let endpoint = format!("{base_url}/mcp"); let initialize = json!({ "jsonrpc": "2.0", "id": 0, @@ -1629,7 +2216,7 @@ impl TestMcpClient { }, }, }); - let response = apply_headers(client.post(&endpoint), &headers) + let response = apply_headers(client.post(endpoint), &headers) .header("content-type", "application/json") .header("accept", "application/json, text/event-stream") .json(&initialize) @@ -1656,7 +2243,7 @@ impl TestMcpClient { Some("2025-06-18") ); - let notification = apply_headers(client.post(&endpoint), &headers) + let notification = apply_headers(client.post(endpoint), &headers) .header("content-type", "application/json") .header("accept", "application/json, text/event-stream") .header("mcp-session-id", &session_id) @@ -1671,7 +2258,7 @@ impl TestMcpClient { Self { client, - endpoint, + endpoint: endpoint.to_string(), session_id, headers, next_id: 1, @@ -1786,53 +2373,37 @@ struct FrameworkMcpHarness { } impl FrameworkMcpHarness { - async fn start(with_actuators: bool) -> Self { + async fn start(with_live_site_runtime: bool) -> Self { let (dir, state, state_path, record) = compile_framework_binding_control_state().await; - let app = test_control_state_app(&dir, state, state_path); - let mut handles = if with_actuators { - install_success_site_actuator(&app).await + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("site controller listener"); + let listen_addr = listener.local_addr().expect("site controller addr"); + let base_url = format!("http://{listen_addr}"); + let app = with_controller_endpoint( + &test_control_state_app(&dir, state, state_path), + &base_url, + listen_addr, + ); + assert_eq!(app.controller_plan.site_id, record.recipient_site_id); + let mut handles = if with_live_site_runtime { + install_framework_site_controller_fixture(&app).await } else { Vec::new() }; - let control_router = Router::new() - .route(CONTROL_SERVICE_PATH, get(get_control_state)) - .route("/v1/control-state/children", post(control_create_child)) - .route( - "/v1/control-state/children/{child}/destroy", - post(control_destroy_child), - ) - .with_state(app.clone()); - let (control_state_url, control_handle) = spawn_test_router(control_router).await; - handles.push(control_handle); - let auth_token = "test-router-auth".to_string(); - let ccs_app = CcsApp { - client: ReqwestClient::new(), - site_state_root: app.state_root.clone(), - control_state_url: Arc::::from(control_state_url), + let controller_app = SiteControllerApp { + control: app, router_auth_token: Arc::::from(auth_token.clone()), - control_state_auth_token: app.control_state_auth_token.clone(), }; - let ccs_router = Router::new() - .nest_service("/mcp", mcp::service(ccs_app.clone())) - .route("/v1/templates", get(ccs_list_templates)) - .route("/v1/templates/{template}", get(ccs_describe_template)) - .route( - "/v1/templates/{template}/resolve", - post(ccs_resolve_template), - ) - .route( - "/v1/children", - get(ccs_list_children).post(ccs_create_child), - ) - .route( - "/v1/children/{child}", - get(ccs_describe_child).delete(ccs_destroy_child), + handles.push(tokio::spawn(async move { + axum::serve( + listener, + super::site_controller::site_controller_router(controller_app).into_make_service(), ) - .route("/v1/snapshot", post(ccs_snapshot)) - .with_state(ccs_app); - let (base_url, ccs_handle) = spawn_test_router(ccs_router).await; - handles.push(ccs_handle); + .await + .expect("site controller should serve"); + })); Self { _dir: dir, @@ -1943,10 +2514,6 @@ async fn framework_component_mcp_discovers_compact_surface() { help.contains("amber.v1.framework_component.inspect"), "help resource should point callers to the inspect tool" ); - assert!( - help.contains("/mcp"), - "help resource should explain the MCP endpoint path" - ); } #[tokio::test] @@ -2101,37 +2668,29 @@ impl DynamicCapsMcpHarness { let state = compile_dynamic_caps_binding_state().await; let state_path = dir.path().join("control-state.json"); write_control_state(&state_path, &state).expect("state should write"); - let app = test_control_state_app(&dir, state, state_path); + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("site controller listener"); + let listen_addr = listener.local_addr().expect("site controller addr"); + let base_url = format!("http://{listen_addr}"); + let app = with_controller_endpoint( + &test_control_state_app(&dir, state, state_path), + &base_url, + listen_addr, + ); let mut handles = vec![install_dynamic_caps_origin_fixture(&app).await]; - let router = Router::new() - .nest_service("/mcp", control_state_mcp::service(app.clone())) - .route( - "/v1/control-state/dynamic-caps/held", - post(control_dynamic_held_list), - ) - .route( - "/v1/control-state/dynamic-caps/held/detail", - post(control_dynamic_held_detail), - ) - .route( - "/v1/control-state/dynamic-caps/share", - post(control_dynamic_share), - ) - .route( - "/v1/control-state/dynamic-caps/inspect-ref", - post(control_dynamic_inspect_ref), - ) - .route( - "/v1/control-state/dynamic-caps/revoke", - post(control_dynamic_revoke), - ) - .route( - "/v1/control-state/dynamic-caps/resolve-origin", - post(control_dynamic_resolve_origin), + let controller_app = SiteControllerApp { + control: app, + router_auth_token: Arc::::from("test-router-auth"), + }; + handles.push(tokio::spawn(async move { + axum::serve( + listener, + super::site_controller::site_controller_router(controller_app).into_make_service(), ) - .with_state(app); - let (base_url, handle) = spawn_test_router(router).await; - handles.push(handle); + .await + .expect("site controller should serve"); + })); Self { _dir: dir, client: Client::new(), @@ -2146,8 +2705,8 @@ impl DynamicCapsMcpHarness { } async fn connect(&self) -> TestMcpClient { - TestMcpClient::connect( - &self.base_url, + TestMcpClient::connect_endpoint( + &format!("{}/v1/controller/dynamic-caps/mcp", self.base_url), "framework-dynamic-caps-test", self.http_headers(), ) @@ -2174,109 +2733,61 @@ impl Drop for DynamicCapsMcpHarness { } async fn install_dynamic_caps_origin_fixture(app: &ControlStateApp) -> tokio::task::JoinHandle<()> { - let site_id = "direct_local"; - let site_state_root = Path::new(&app.state_root).join(site_id); - let artifact_dir = site_state_root.join("artifact"); - let runtime_root = site_state_root.join("runtime"); - fs::create_dir_all(artifact_dir.join(".amber")).expect("artifact root should exist"); + let plan = app.controller_plan.as_ref(); + let site_state_root = PathBuf::from(&plan.site_state_root); + let artifact_dir = PathBuf::from(&plan.artifact_dir); + let runtime_root = PathBuf::from( + plan.runtime_root + .as_deref() + .expect("test controller plan should include runtime root"), + ); + fs::create_dir_all(artifact_dir.join(".amber").join("control")) + .expect("artifact control dir should exist"); fs::create_dir_all(&runtime_root).expect("runtime root should exist"); + fs::create_dir_all(&site_state_root).expect("site state root should exist"); - let control_state_auth_token = app.control_state_auth_token.to_string(); - let publish_router = Router::new().route( - "/v1/internal/dynamic-caps/origins/publish", - post({ - let expected = control_state_auth_token.clone(); - move |headers: HeaderMap, - Json(request): Json| { - let expected = expected.clone(); - async move { - assert_eq!( - headers - .get(FRAMEWORK_AUTH_HEADER) - .and_then(|value| value.to_str().ok()), - Some(expected.as_str()), - "origin publish should authenticate with the control-state token", - ); - Json(dynamic_caps::PublishDynamicCapabilityOriginResponse { - route_id: request.route_id, - capability: "provider.http".to_string(), - protocol: "http".to_string(), - }) - } - } - }), + write_json(&site_controller_plan_path(&site_state_root), plan) + .expect("site controller plan should write"); + + let overlay_router = Router::new().route( + "/overlays/{overlay_id}", + axum::routing::put(|| async { StatusCode::NO_CONTENT }) + .delete(|| async { StatusCode::NO_CONTENT }), ); - let (publish_base_url, publish_handle) = spawn_test_router(publish_router).await; - let publish_addr = publish_base_url + let (router_base_url, overlay_handle) = spawn_test_router(overlay_router).await; + let router_control = router_base_url .strip_prefix("http://") - .expect("publish base URL should be absolute HTTP") - .parse() - .expect("publish base URL should parse as socket address"); + .expect("router control URL should be absolute HTTP") + .to_string(); write_json( - &site_state_root.join("site-actuator-plan.json"), - &SiteActuatorPlan { - schema: "amber.run.site_actuator_plan".to_string(), - version: 1, - run_id: "test-run".to_string(), - mesh_scope: "test-mesh".to_string(), - run_root: app.run_root.display().to_string(), - site_id: site_id.to_string(), - kind: SiteKind::Direct, - router_identity_id: "/router".to_string(), - artifact_dir: artifact_dir.display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr: "127.0.0.1:0".parse().expect("listen addr"), - storage_root: None, - runtime_root: Some(runtime_root.display().to_string()), - router_mesh_port: None, - compose_project: None, - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), - }, - ) - .expect("site actuator plan should write"); - write_json( - &site_state_root.join("framework-ccs-plan.json"), - &FrameworkCcsPlan { - schema: CCS_PLAN_SCHEMA.to_string(), - version: CCS_PLAN_VERSION, - site_id: site_id.to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr: publish_addr, - control_state_url: "http://127.0.0.1:0".to_string(), - router_auth_token: "unused".to_string(), - control_state_auth_token: control_state_auth_token.clone(), - }, - ) - .expect("framework ccs plan should write"); - write_json( - &site_state_root.join("manager-state.json"), + &site_state_path(&app.state_root, &plan.site_id), &json!({ "status": "running", - "kind": SiteKind::Direct, + "kind": plan.kind, "artifact_dir": artifact_dir.display().to_string(), "supervisor_pid": 1, + "router_control": router_control, "router_mesh_addr": "127.0.0.1:39001", - "router_identity_id": "/router", + "router_identity_id": plan.router_identity_id, "router_public_key_b64": "dGVzdC1yb3V0ZXIta2V5", + "site_controller_pid": 1, + "site_controller_url": plan.authority_url, }), ) .expect("manager state should write"); write_json( - &site_state_root.join("site-actuator-state.json"), + &site_state_root.join("site-controller-runtime-state.json"), &json!({ - "schema": "amber.site_actuator_state", + "schema": "amber.run.site_controller_runtime_state", "version": 1, - "run_id": "test-run", - "site_id": site_id, - "kind": SiteKind::Direct, + "run_id": plan.run_id, + "site_id": plan.site_id, + "kind": plan.kind, "children": {}, }), ) - .expect("site actuator state should write"); + .expect("site controller runtime state should write"); write_json( &artifact_dir.join("direct-plan.json"), &json!({ @@ -2324,7 +2835,7 @@ async fn install_dynamic_caps_origin_fixture(app: &ControlStateApp) -> tokio::ta }, ], "router": { - "identity_id": "/router", + "identity_id": plan.router_identity_id, "mesh_port": 39001, "control_port": 39011, "control_socket_path": "router.sock", @@ -2335,8 +2846,8 @@ async fn install_dynamic_caps_origin_fixture(app: &ControlStateApp) -> tokio::ta ) .expect("direct plan should write"); write_json( - &crate::direct_runtime::direct_runtime_state_path(&artifact_dir), - &crate::direct_runtime::DirectRuntimeState { + &direct_runtime_state_path(&artifact_dir), + &DirectRuntimeState { component_mesh_port_by_id: BTreeMap::from([(1, 24001), (2, 24002)]), ..Default::default() }, @@ -2361,18 +2872,31 @@ async fn install_dynamic_caps_origin_fixture(app: &ControlStateApp) -> tokio::ta "/alice", "127.0.0.1:24002", Vec::new(), - Vec::new(), + vec![OutboundRoute { + route_id: "provider-route".to_string(), + slot: "upstream".to_string(), + capability_kind: Some("http".to_string()), + capability_profile: None, + listen_port: 20000, + listen_addr: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + peer_addr: "127.0.0.1:24001".to_string(), + peer_id: "/provider".to_string(), + capability: "http".to_string(), + }], ) .mesh_config, ) .expect("alice mesh config should write"); - write_json( - &runtime_root.join("router-mesh.json"), - &test_live_site_router(Vec::new()), - ) + write_json(&runtime_root.join("router-mesh.json"), &{ + let mut router = test_live_site_router(Vec::new()); + router.identity.id = plan.router_identity_id.clone(); + router + }) .expect("router mesh config should write"); - publish_handle + overlay_handle } #[tokio::test] @@ -2412,10 +2936,6 @@ async fn dynamic_caps_mcp_discovers_compact_surface() { help.contains("amber.v1.framework_dynamic_caps.inspect"), "help resource should point callers to the inspect tool" ); - assert!( - help.contains("/v1/control-state/dynamic-caps"), - "help resource should explain the HTTP endpoint family", - ); } #[tokio::test] @@ -2428,7 +2948,7 @@ async fn dynamic_caps_mcp_matches_http_surface() { holder_component_id: "components./alice".to_string(), }; let http_held: amber_mesh::dynamic_caps::HeldListResponse = http - .post_json("/v1/control-state/dynamic-caps/held", &held_list_request) + .post_json("/v1/controller/dynamic-caps/held", &held_list_request) .await; let mcp_held: Value = mcp .call_tool( @@ -2456,7 +2976,7 @@ async fn dynamic_caps_mcp_matches_http_surface() { }; let http_detail: HeldEntryDetail = http .post_json( - "/v1/control-state/dynamic-caps/held/detail", + "/v1/controller/dynamic-caps/held/detail", &held_detail_request, ) .await; @@ -2490,7 +3010,7 @@ async fn dynamic_caps_mcp_matches_http_surface() { options: Value::Null, }; let http_share: amber_mesh::dynamic_caps::ShareResponse = http - .post_json("/v1/control-state/dynamic-caps/share", &share_request) + .post_json("/v1/controller/dynamic-caps/share", &share_request) .await; let mcp_share: Value = mcp .call_tool( @@ -2531,7 +3051,7 @@ async fn dynamic_caps_mcp_matches_http_surface() { holder_component_id: "components./carol".to_string(), }; let http_carol_held: amber_mesh::dynamic_caps::HeldListResponse = http - .post_json("/v1/control-state/dynamic-caps/held", &carol_held_request) + .post_json("/v1/controller/dynamic-caps/held", &carol_held_request) .await; let mcp_carol_held: Value = mcp .call_tool( @@ -2553,7 +3073,7 @@ async fn dynamic_caps_mcp_matches_http_surface() { }; let http_inspect_ref: amber_mesh::dynamic_caps::InspectRefResponse = http .post_json( - "/v1/control-state/dynamic-caps/inspect-ref", + "/v1/controller/dynamic-caps/inspect-ref", &inspect_ref_request, ) .await; @@ -2580,7 +3100,7 @@ async fn dynamic_caps_mcp_matches_http_surface() { }; let http_resolve_origin: dynamic_caps::ControlDynamicResolveOriginResponse = http .post_json( - "/v1/control-state/dynamic-caps/resolve-origin", + "/v1/controller/dynamic-caps/resolve-origin", &resolve_origin_request, ) .await; @@ -2610,7 +3130,7 @@ async fn dynamic_caps_mcp_matches_http_surface() { }, }; let http_revoke: amber_mesh::dynamic_caps::RevokeResponse = http - .post_json("/v1/control-state/dynamic-caps/revoke", &revoke_request) + .post_json("/v1/controller/dynamic-caps/revoke", &revoke_request) .await; let mcp_revoke: Value = mcp .call_tool( @@ -2636,7 +3156,7 @@ async fn dynamic_caps_mcp_matches_http_surface() { }; let http_revoked_detail: HeldEntryDetail = http .post_json( - "/v1/control-state/dynamic-caps/held/detail", + "/v1/controller/dynamic-caps/held/detail", &revoked_detail_request, ) .await; @@ -2656,7 +3176,44 @@ async fn dynamic_caps_mcp_matches_http_surface() { ); } -async fn install_success_site_actuator(app: &ControlStateApp) -> Vec> { +#[derive(Clone)] +struct MockSiteControllerState { + expected_auth: String, + rollback_status: StatusCode, +} + +async fn mock_site_controller_status( + State(state): State, + headers: HeaderMap, +) -> StatusCode { + assert_eq!( + headers + .get(FRAMEWORK_AUTH_HEADER) + .and_then(|value| value.to_str().ok()), + Some(state.expected_auth.as_str()), + "site controller requests should authenticate with the control token", + ); + StatusCode::NO_CONTENT +} + +async fn mock_site_controller_rollback_status( + State(state): State, + headers: HeaderMap, +) -> StatusCode { + assert_eq!( + headers + .get(FRAMEWORK_AUTH_HEADER) + .and_then(|value| value.to_str().ok()), + Some(state.expected_auth.as_str()), + "site controller requests should authenticate with the control token", + ); + state.rollback_status +} + +async fn install_mock_site_controllers( + app: &ControlStateApp, + rollback_status: StatusCode, +) -> (ControlStateApp, Vec>) { let offered_sites = { let state = app.control_state.lock().await; state @@ -2666,139 +3223,83 @@ async fn install_success_site_actuator(app: &ControlStateApp) -> Vec>() }; + let mut peer_controllers = BTreeMap::new(); let mut handles = Vec::with_capacity(offered_sites.len()); for (site_id, site_kind) in offered_sites { - let site_state_root = Path::new(&app.state_root).join(&site_id); - fs::create_dir_all(&site_state_root).expect("site state root should exist"); - let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) - .await - .expect("actuator listener"); - let listen_addr = listener.local_addr().expect("actuator addr"); - write_json( - &site_state_root.join("site-actuator-plan.json"), - &SiteActuatorPlan { - schema: "amber.run.site_actuator_plan".to_string(), - version: 1, - run_id: "test-run".to_string(), - mesh_scope: "test-mesh".to_string(), - run_root: app.run_root.display().to_string(), - site_id: site_id.clone(), - kind: site_kind, - router_identity_id: format!("/site/{site_id}/router"), - artifact_dir: site_state_root.join("artifact").display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr, - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: None, - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), - }, - ) - .expect("site actuator plan should write"); - let app = Router::new() + let state = MockSiteControllerState { + expected_auth: app.control_state_auth_token.to_string(), + rollback_status, + }; + let router = Router::new() .route( - "/v1/children/{child_id}/prepare", - post(|| async { StatusCode::NO_CONTENT }), + "/v1/controller/site/children/{child_id}/prepare", + post(mock_site_controller_status), ) .route( - "/v1/children/{child_id}/rollback", - post(|| async { StatusCode::NO_CONTENT }), + "/v1/controller/site/children/{child_id}/rollback", + post(mock_site_controller_rollback_status), ) .route( - "/v1/children/{child_id}/publish", - post(|| async { StatusCode::NO_CONTENT }), + "/v1/controller/site/children/{child_id}/publish", + post(mock_site_controller_status), ) .route( - "/v1/children/{child_id}/destroy", - post(|| async { StatusCode::NO_CONTENT }), - ); - handles.push(tokio::spawn(async move { - axum::serve(listener, app) - .await - .expect("site actuator should serve"); - })); + "/v1/controller/site/children/{child_id}/destroy", + post(mock_site_controller_status), + ) + .with_state(state); + let (authority_url, handle) = spawn_test_router(router).await; + peer_controllers.insert( + site_id.clone(), + SiteControllerPeerPlan { + site_id, + kind: site_kind, + authority_url, + }, + ); + handles.push(handle); } - handles + (with_peer_site_controllers(app, peer_controllers), handles) } -async fn install_failing_rollback_site_actuator( +async fn install_success_site_controllers( app: &ControlStateApp, -) -> Vec> { - let offered_sites = { - let state = app.control_state.lock().await; - state - .placement - .offered_sites - .iter() - .map(|(site_id, site)| (site_id.clone(), site.kind)) - .collect::>() - }; - let mut handles = Vec::with_capacity(offered_sites.len()); - for (site_id, site_kind) in offered_sites { - let site_state_root = Path::new(&app.state_root).join(&site_id); - fs::create_dir_all(&site_state_root).expect("site state root should exist"); - let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) - .await - .expect("actuator listener"); - let listen_addr = listener.local_addr().expect("actuator addr"); - write_json( - &site_state_root.join("site-actuator-plan.json"), - &SiteActuatorPlan { - schema: "amber.run.site_actuator_plan".to_string(), - version: 1, - run_id: "test-run".to_string(), - mesh_scope: "test-mesh".to_string(), - run_root: app.run_root.display().to_string(), - site_id: site_id.clone(), - kind: site_kind, - router_identity_id: format!("/site/{site_id}/router"), - artifact_dir: site_state_root.join("artifact").display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr, - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: None, - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), - }, - ) - .expect("site actuator plan should write"); - let app = Router::new() - .route( - "/v1/children/{child_id}/prepare", - post(|| async { StatusCode::NO_CONTENT }), - ) - .route( - "/v1/children/{child_id}/rollback", - post(|| async { StatusCode::INTERNAL_SERVER_ERROR }), - ) - .route( - "/v1/children/{child_id}/publish", - post(|| async { StatusCode::NO_CONTENT }), - ) - .route( - "/v1/children/{child_id}/destroy", - post(|| async { StatusCode::NO_CONTENT }), - ); - handles.push(tokio::spawn(async move { - axum::serve(listener, app) - .await - .expect("site actuator should serve"); - })); +) -> (ControlStateApp, Vec>) { + install_mock_site_controllers(app, StatusCode::NO_CONTENT).await +} + +async fn install_failing_rollback_site_controllers( + app: &ControlStateApp, +) -> (ControlStateApp, Vec>) { + install_mock_site_controllers(app, StatusCode::INTERNAL_SERVER_ERROR).await +} + +fn with_peer_site_controllers( + app: &ControlStateApp, + peer_controllers: BTreeMap, +) -> ControlStateApp { + let mut controller_plan = app.controller_plan.as_ref().clone(); + controller_plan.site_id = "__test_controller__".to_string(); + controller_plan.peer_controllers = peer_controllers.clone(); + ControlStateApp { + control_state: app.control_state.clone(), + client: app.client.clone(), + state_path: app.state_path.clone(), + run_root: app.run_root.clone(), + state_root: app.state_root.clone(), + mesh_scope: app.mesh_scope.clone(), + control_state_auth_token: app.control_state_auth_token.clone(), + controller_plan: Arc::new(controller_plan), + peer_controllers: Arc::new(peer_controllers), + authority_locks: app.authority_locks.clone(), + runtime: app.runtime.clone(), } - handles } -async fn install_barrier_destroy_site_actuator( +async fn install_barrier_destroy_site_controllers( app: &ControlStateApp, ) -> ( + ControlStateApp, Vec>, tokio::sync::mpsc::UnboundedReceiver, Arc, @@ -2814,78 +3315,73 @@ async fn install_barrier_destroy_site_actuator( }; let barrier = Arc::new(tokio::sync::Barrier::new(offered_sites.len() + 1)); let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let mut peer_controllers = BTreeMap::new(); let mut handles = Vec::with_capacity(offered_sites.len()); for (site_id, site_kind) in offered_sites { - let site_state_root = Path::new(&app.state_root).join(&site_id); - fs::create_dir_all(&site_state_root).expect("site state root should exist"); - let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) - .await - .expect("actuator listener"); - let listen_addr = listener.local_addr().expect("actuator addr"); - write_json( - &site_state_root.join("site-actuator-plan.json"), - &SiteActuatorPlan { - schema: "amber.run.site_actuator_plan".to_string(), - version: 1, - run_id: "test-run".to_string(), - mesh_scope: "test-mesh".to_string(), - run_root: app.run_root.display().to_string(), - site_id: site_id.clone(), - kind: site_kind, - router_identity_id: format!("/site/{site_id}/router"), - artifact_dir: site_state_root.join("artifact").display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr, - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: None, - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), - }, - ) - .expect("site actuator plan should write"); + let state = MockSiteControllerState { + expected_auth: app.control_state_auth_token.to_string(), + rollback_status: StatusCode::NO_CONTENT, + }; let start_tx = tx.clone(); let destroy_barrier = barrier.clone(); let site_id_for_destroy = site_id.clone(); - let app = Router::new() + let router = Router::new() .route( - "/v1/children/{child_id}/prepare", - post(|| async { StatusCode::NO_CONTENT }), + "/v1/controller/site/children/{child_id}/prepare", + post(mock_site_controller_status), ) .route( - "/v1/children/{child_id}/publish", - post(|| async { StatusCode::NO_CONTENT }), + "/v1/controller/site/children/{child_id}/publish", + post(mock_site_controller_status), ) .route( - "/v1/children/{child_id}/destroy", - post(move || { - let start_tx = start_tx.clone(); - let destroy_barrier = destroy_barrier.clone(); - let site_id = site_id_for_destroy.clone(); - async move { - start_tx - .send(site_id) - .expect("destroy start notification should send"); - destroy_barrier.wait().await; - StatusCode::NO_CONTENT - } - }), + "/v1/controller/site/children/{child_id}/destroy", + post( + move |State(state): State, headers: HeaderMap| { + let start_tx = start_tx.clone(); + let destroy_barrier = destroy_barrier.clone(); + let site_id = site_id_for_destroy.clone(); + async move { + assert_eq!( + headers + .get(FRAMEWORK_AUTH_HEADER) + .and_then(|value| value.to_str().ok()), + Some(state.expected_auth.as_str()), + "site controller requests should authenticate with the control \ + token", + ); + start_tx + .send(site_id) + .expect("destroy start notification should send"); + destroy_barrier.wait().await; + StatusCode::NO_CONTENT + } + }, + ), ); - handles.push(tokio::spawn(async move { - axum::serve(listener, app) - .await - .expect("site actuator should serve"); - })); + let (authority_url, handle) = spawn_test_router(router.with_state(state)).await; + peer_controllers.insert( + site_id.clone(), + SiteControllerPeerPlan { + site_id, + kind: site_kind, + authority_url, + }, + ); + handles.push(handle); } - (handles, rx, barrier) + ( + with_peer_site_controllers(app, peer_controllers), + handles, + rx, + barrier, + ) } -async fn install_barrier_publish_site_actuator( +async fn install_barrier_publish_site_controllers( app: &ControlStateApp, ) -> ( + ControlStateApp, Vec>, tokio::sync::mpsc::UnboundedReceiver, Arc, @@ -2901,73 +3397,67 @@ async fn install_barrier_publish_site_actuator( }; let barrier = Arc::new(tokio::sync::Barrier::new(offered_sites.len() + 1)); let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let mut peer_controllers = BTreeMap::new(); let mut handles = Vec::with_capacity(offered_sites.len()); for (site_id, site_kind) in offered_sites { - let site_state_root = Path::new(&app.state_root).join(&site_id); - fs::create_dir_all(&site_state_root).expect("site state root should exist"); - let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) - .await - .expect("actuator listener"); - let listen_addr = listener.local_addr().expect("actuator addr"); - write_json( - &site_state_root.join("site-actuator-plan.json"), - &SiteActuatorPlan { - schema: "amber.run.site_actuator_plan".to_string(), - version: 1, - run_id: "test-run".to_string(), - mesh_scope: "test-mesh".to_string(), - run_root: app.run_root.display().to_string(), - site_id: site_id.clone(), - kind: site_kind, - router_identity_id: format!("/site/{site_id}/router"), - artifact_dir: site_state_root.join("artifact").display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr, - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: None, - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), - }, - ) - .expect("site actuator plan should write"); + let state = MockSiteControllerState { + expected_auth: app.control_state_auth_token.to_string(), + rollback_status: StatusCode::NO_CONTENT, + }; let start_tx = tx.clone(); let publish_barrier = barrier.clone(); let site_id_for_publish = site_id.clone(); - let app = Router::new() + let router = Router::new() .route( - "/v1/children/{child_id}/prepare", - post(|| async { StatusCode::NO_CONTENT }), + "/v1/controller/site/children/{child_id}/prepare", + post(mock_site_controller_status), ) .route( - "/v1/children/{child_id}/publish", - post(move || { - let start_tx = start_tx.clone(); - let publish_barrier = publish_barrier.clone(); - let site_id = site_id_for_publish.clone(); - async move { - start_tx - .send(site_id) - .expect("publish start notification should send"); - publish_barrier.wait().await; - StatusCode::NO_CONTENT - } - }), + "/v1/controller/site/children/{child_id}/publish", + post( + move |State(state): State, headers: HeaderMap| { + let start_tx = start_tx.clone(); + let publish_barrier = publish_barrier.clone(); + let site_id = site_id_for_publish.clone(); + async move { + assert_eq!( + headers + .get(FRAMEWORK_AUTH_HEADER) + .and_then(|value| value.to_str().ok()), + Some(state.expected_auth.as_str()), + "site controller requests should authenticate with the control \ + token", + ); + start_tx + .send(site_id) + .expect("publish start notification should send"); + publish_barrier.wait().await; + StatusCode::NO_CONTENT + } + }, + ), ) .route( - "/v1/children/{child_id}/destroy", - post(|| async { StatusCode::NO_CONTENT }), + "/v1/controller/site/children/{child_id}/destroy", + post(mock_site_controller_status), ); - handles.push(tokio::spawn(async move { - axum::serve(listener, app) - .await - .expect("site actuator should serve"); - })); + let (authority_url, handle) = spawn_test_router(router.with_state(state)).await; + peer_controllers.insert( + site_id.clone(), + SiteControllerPeerPlan { + site_id, + kind: site_kind, + authority_url, + }, + ); + handles.push(handle); } - (handles, rx, barrier) + ( + with_peer_site_controllers(app, peer_controllers), + handles, + rx, + barrier, + ) } #[tokio::test] @@ -3338,6 +3828,49 @@ async fn open_template_replay_uses_admitted_manifest_after_source_mutation() { fs::remove_file(&beta_path).expect("beta source should be removable before replay"); fs::remove_file(&beta_leaf_path).expect("beta leaf source should be removable before replay"); + let replay_scenario_ir: ScenarioIr = serde_json::from_value(snapshot_response.scenario.clone()) + .expect("snapshot scenario should decode for replay run plan"); + let replay_placement = placement_from_snapshot(&snapshot_response); + let replay_compiled = + CompiledScenario::from_ir(replay_scenario_ir).expect("snapshot replay should compile"); + let replay_run_plan = + build_run_plan(&replay_compiled, Some(&replay_placement)).expect("replay run plan"); + let mut localized_replay = build_site_controller_state( + "replay-run", + &replay_run_plan, + "direct_local", + 0, + 1, + &mesh_dynamic_caps::signing_seed_b64(&mesh_dynamic_caps::signing_key_from_seed( + mesh_dynamic_caps::generate_dynamic_capability_signing_seed(), + )), + ) + .expect("site-local replay state should build"); + assert!( + localized_replay + .live_children + .iter() + .any(|child| child.name == "job-open" && child.state == ChildState::Live), + "site-local replay should preserve live children even when the root authority has no \ + explicit assignment" + ); + let replayed_live_child_id = localized_replay + .live_children + .iter() + .map(|child| child.child_id) + .max() + .expect("site-local replay should keep the restored live child id"); + assert_eq!( + localized_replay.next_child_id, replayed_live_child_id, + "site-local replay must preserve the child id allocator after restoring live children" + ); + let replay_next_child_id = allocate_child_id(&mut localized_replay); + assert!( + replay_next_child_id > replayed_live_child_id, + "site-local replay must allocate a fresh child id instead of colliding with restored \ + children" + ); + let mut replayed = compile_control_state_from_snapshot(&snapshot_response).await; let replay_state_path = dir.path().join("replay-control-state.json"); write_control_state(&replay_state_path, &replayed).expect("replay state should write"); @@ -4185,6 +4718,7 @@ fn shared_cross_site_link_is_retained_while_another_child_still_needs_it() { generation: 0, next_child_id: 2, next_tx_id: 0, + id_stride: 1, next_component_id: 0, dynamic_capability_signing_seed_b64: mesh_dynamic_caps::signing_seed_b64( &mesh_dynamic_caps::signing_key_from_seed( @@ -4304,6 +4838,73 @@ async fn create_rejects_duplicate_names_and_destroy_is_idempotent() { ); } +#[tokio::test] +async fn create_aborts_and_destroys_partially_published_child_when_publish_fails() { + let (dir, state, state_path) = compile_exact_template_control_state().await; + let authority_realm_id = state.base_scenario.root; + let base_app = test_control_state_app(&dir, state, state_path); + let runtime = Arc::new(FailingPublishRuntime::default()); + let app = with_runtime(&base_app, runtime.clone()); + + let err = execute_create_child( + &app, + authority_realm_id, + CreateChildRequest { + template: "worker".to_string(), + name: "broken".to_string(), + manifest: None, + config: BTreeMap::new(), + bindings: BTreeMap::new(), + }, + ) + .await + .expect_err("publish failure should abort the create"); + assert_eq!(err.0.code, ProtocolErrorCode::PublishFailed); + assert!( + err.0.message.contains("publish exploded"), + "publish failure should preserve the underlying detail, got: {}", + err.0.message + ); + + let state = app.control_state.lock().await.clone(); + assert!( + state.pending_creates.is_empty(), + "failed publish cleanup must remove the pending create" + ); + assert!( + state + .live_children + .iter() + .all(|child| child.name != "broken"), + "failed publish cleanup must not leave a live child behind" + ); + assert_eq!( + state.journal.last().map(|entry| entry.state), + Some(ChildState::CreateAborted), + "failed publish cleanup must record an aborted transaction" + ); + + let destroy_calls = runtime + .destroy_calls + .lock() + .expect("destroy call log mutex should lock") + .clone(); + assert_eq!( + destroy_calls.len(), + 1, + "failed publish cleanup must destroy the prepared site-local child" + ); + assert_eq!(destroy_calls[0].0, 1); + assert_eq!( + destroy_calls[0] + .1 + .as_ref() + .map(|site_plan| site_plan.site_id.as_str()), + Some("direct_local"), + "failed publish cleanup must reconcile the child site back to the desired plan", + ); +} + #[tokio::test] async fn max_live_children_is_scoped_per_template() { let dir = TempDir::new().expect("temp dir"); @@ -4562,7 +5163,7 @@ async fn concurrent_same_name_creates_serialize_to_one_live_child() { let (dir, state, state_path) = compile_exact_template_control_state().await; let root_authority = state.base_scenario.root; let app = test_control_state_app(&dir, state, state_path); - let actuators = install_success_site_actuator(&app).await; + let (app, controllers) = install_success_site_controllers(&app).await; let request = CreateChildRequest { template: "worker".to_string(), name: "job".to_string(), @@ -4611,8 +5212,8 @@ async fn concurrent_same_name_creates_serialize_to_one_live_child() { 1, "snapshot should remain clean after the same-name race", ); - for actuator in actuators { - actuator.abort(); + for controller in controllers { + controller.abort(); } } @@ -4667,8 +5268,8 @@ async fn destroy_retracted_tears_down_sites_concurrently() { write_control_state(&state_path, &state).expect("state should write"); let root_authority = state.base_scenario.root; let app = test_control_state_app(&dir, state, state_path); - let (actuators, mut destroy_starts, barrier) = - install_barrier_destroy_site_actuator(&app).await; + let (app, controllers, mut destroy_starts, barrier) = + install_barrier_destroy_site_controllers(&app).await; { let mut state = app.control_state.lock().await; state.pending_destroys.push(pending_destroy( @@ -4729,7 +5330,7 @@ async fn destroy_retracted_tears_down_sites_concurrently() { .expect("second destroy notification should arrive"); assert_ne!( first, second, - "destroy should reach both site actuators before either completes" + "destroy should reach both peer site controllers before either completes" ); barrier.wait().await; @@ -4743,8 +5344,8 @@ async fn destroy_retracted_tears_down_sites_concurrently() { recovered.live_children.is_empty(), "successful destroy should remove the child after concurrent site teardown", ); - for actuator in actuators { - actuator.abort(); + for controller in controllers { + controller.abort(); } } @@ -4863,8 +5464,8 @@ async fn create_committed_hidden_publishes_independent_sites_concurrently() { ]; write_control_state(&state_path, &state).expect("state should write"); let app = test_control_state_app(&dir, state, state_path); - let (actuators, mut publish_starts, barrier) = - install_barrier_publish_site_actuator(&app).await; + let (app, controllers, mut publish_starts, barrier) = + install_barrier_publish_site_controllers(&app).await; { let mut state = app.control_state.lock().await; state.pending_creates.push(pending_create(1, child)); @@ -4885,7 +5486,7 @@ async fn create_committed_hidden_publishes_independent_sites_concurrently() { .expect("second publish notification should arrive"); assert_ne!( first, second, - "create should reach both independent site actuators before either completes" + "create should reach both independent site controllers before either completes" ); barrier.wait().await; @@ -4905,8 +5506,8 @@ async fn create_committed_hidden_publishes_independent_sites_concurrently() { ChildState::Live, "successful concurrent site publication should promote the child to live", ); - for actuator in actuators { - actuator.abort(); + for controller in controllers { + controller.abort(); } } @@ -4970,7 +5571,7 @@ async fn concurrent_distinct_creates_commit_both_children() { write_control_state(&state_path, &state).expect("state should write"); let root_authority = state.base_scenario.root; let app = test_control_state_app(&dir, state, state_path); - let actuators = install_success_site_actuator(&app).await; + let (app, controllers) = install_success_site_controllers(&app).await; let (left, right) = tokio::join!( execute_create_child( @@ -5031,8 +5632,8 @@ async fn concurrent_distinct_creates_commit_both_children() { .any(|component| component.moniker == "/job-b"), "snapshot should contain the second child", ); - for actuator in actuators { - actuator.abort(); + for controller in controllers { + controller.abort(); } } @@ -6031,7 +6632,7 @@ async fn recover_control_state_surfaces_create_prepared_rollback_failures() { let state_path = dir.path().join("control-state.json"); let root_authority = state.base_scenario.root; let app = test_control_state_app(&dir, state.clone(), state_path.clone()); - let actuators = install_failing_rollback_site_actuator(&app).await; + let (app, controllers) = install_failing_rollback_site_controllers(&app).await; state.pending_creates.push(pending_create( 1, LiveChildRecord { @@ -6082,8 +6683,8 @@ async fn recover_control_state_surfaces_create_prepared_rollback_failures() { recovered.journal.is_empty(), "failed rollback must not pretend the child was aborted" ); - for actuator in actuators { - actuator.abort(); + for controller in controllers { + controller.abort(); } } @@ -6194,8 +6795,8 @@ async fn recover_control_state_does_not_republish_live_children() { }); write_control_state(&state_path, &state).expect("state should write"); let app = test_control_state_app(&dir, state, state_path); - let (actuators, mut publish_starts, _barrier) = - install_barrier_publish_site_actuator(&app).await; + let (_app, controllers, mut publish_starts, _barrier) = + install_barrier_publish_site_controllers(&app).await; recover_control_state(&app) .await @@ -6215,8 +6816,8 @@ async fn recover_control_state_does_not_republish_live_children() { recovered.journal.is_empty(), "live recovery should not append synthetic journal entries", ); - for actuator in actuators { - actuator.abort(); + for controller in controllers { + controller.abort(); } } From 09da7d86c99c0be1d6803dbaaca72e26e9f57730 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Mon, 13 Apr 2026 08:15:59 +0800 Subject: [PATCH 02/25] Refactor framework.component and dynamic caps --- Cargo.lock | 35 + cli/Cargo.toml | 1 + cli/src/direct_runtime/preview.rs | 104 +- cli/src/direct_runtime/runtime.rs | 219 +- cli/src/main.rs | 6 +- cli/src/mixed_run.rs | 6551 +---------------- cli/src/mixed_run/launch_bundle.rs | 293 +- cli/src/mixed_run/supervisor.rs | 110 +- cli/src/mixed_run/tests.rs | 4126 +---------- cli/src/vm_runtime.rs | 5 +- cli/src/vm_runtime/state.rs | 51 + cli/tests/linux_vm.rs | 16 + cli/tests/mixed_run/support.rs | 7 +- compiler/src/run_plan.rs | 127 +- compiler/src/targets/direct/mod.rs | 30 +- .../src/targets/mesh/docker_compose/mod.rs | 107 +- .../mesh/docker_compose/tests/rendering.rs | 73 + compiler/src/targets/mesh/internal_images.rs | 83 +- compiler/src/targets/mesh/kubernetes/mod.rs | 27 +- compiler/src/targets/mesh/mesh_config.rs | 25 +- compiler/src/targets/vm/mod.rs | 30 +- docker/amber-site-controller/Dockerfile | 84 + docker/images.json | 6 + examples/observability-debug/README.md | 162 +- images/src/lib.rs | 136 + runtime/proxy/src/tests.rs | 8 +- runtime/site-controller/Cargo.toml | 3 + .../src/bin/amber-site-controller.rs | 55 +- .../site-controller/src/control_state_api.rs | 144 +- .../site-controller/src/default_runtime.rs | 1907 +++++ .../src/default_runtime/child_runtime.rs | 1139 +++ .../src/default_runtime/compose_controller.rs | 80 + .../src/default_runtime/dynamic_routes.rs | 1696 +++++ .../default_runtime/kubernetes_controller.rs | 464 ++ .../src/default_runtime/site_artifacts.rs | 2435 ++++++ .../default_runtime/site_runtime_support.rs | 1235 ++++ runtime/site-controller/src/dynamic_caps.rs | 12 + runtime/site-controller/src/lib.rs | 35 +- runtime/site-controller/src/orchestration.rs | 1781 +++-- runtime/site-controller/src/planner.rs | 507 +- runtime/site-controller/src/runtime_api.rs | 32 +- .../site-controller/src/site_controller.rs | 642 +- runtime/site-controller/src/state.rs | 63 +- runtime/site-controller/src/tests.rs | 2047 ++--- 44 files changed, 13402 insertions(+), 13297 deletions(-) create mode 100644 docker/amber-site-controller/Dockerfile create mode 100644 runtime/site-controller/src/default_runtime.rs create mode 100644 runtime/site-controller/src/default_runtime/child_runtime.rs create mode 100644 runtime/site-controller/src/default_runtime/compose_controller.rs create mode 100644 runtime/site-controller/src/default_runtime/dynamic_routes.rs create mode 100644 runtime/site-controller/src/default_runtime/kubernetes_controller.rs create mode 100644 runtime/site-controller/src/default_runtime/site_artifacts.rs create mode 100644 runtime/site-controller/src/default_runtime/site_runtime_support.rs diff --git a/Cargo.lock b/Cargo.lock index aa6d20cf..dd7cdba6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -428,6 +428,7 @@ dependencies = [ "amber-scenario", "axum", "base64", + "libc", "miette", "regex", "reqwest 0.13.2", @@ -435,6 +436,8 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sha2", + "tar", "tempfile", "tokio", "url", @@ -1472,6 +1475,17 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -4153,6 +4167,17 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "tar" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.26.0" @@ -5415,6 +5440,16 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "yoke" version = "0.8.1" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 1e60f126..854fabcd 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -10,6 +10,7 @@ path = "src/main.rs" amber-compiler = { workspace = true } amber-config = { workspace = true } amber-helper = { path = "../runtime/helper" } +amber-images = { workspace = true } amber-manifest = { workspace = true } amber-mesh = { workspace = true } amber-proxy = { workspace = true } diff --git a/cli/src/direct_runtime/preview.rs b/cli/src/direct_runtime/preview.rs index 2f66cc1f..d3c89dea 100644 --- a/cli/src/direct_runtime/preview.rs +++ b/cli/src/direct_runtime/preview.rs @@ -120,27 +120,64 @@ pub(crate) fn build_direct_site_launch_preview( direct_plan, mesh_plan, } = load_direct_runtime_inputs(&plan_path)?; - let runtime_state = materialize_direct_runtime( + let mut inspectability_warnings = Vec::new(); + let runtime_state = match materialize_direct_runtime( &plan_root, runtime_root, &direct_plan, &mesh_plan, router_mesh_port, true, - )?; - #[cfg(target_os = "linux")] - configure_direct_mesh_network(runtime_root, &runtime_state, &direct_plan)?; - #[cfg(not(target_os = "linux"))] - configure_direct_mesh_network(runtime_root, &runtime_state, &direct_plan)?; + ) { + Ok(runtime_state) => { + #[cfg(target_os = "linux")] + configure_direct_mesh_network(runtime_root, &runtime_state, &direct_plan)?; + #[cfg(not(target_os = "linux"))] + configure_direct_mesh_network(runtime_root, &runtime_state, &direct_plan)?; + runtime_state + } + Err(err) if missing_existing_peer_identity(&err) => { + // Dry-run preview can still show the local process shape before peer-site routers + // exist; the missing identities only block full mesh config materialization. + inspectability_warnings.push(format!( + "preview is missing one or more peer-site router identities, so mesh configs \ + remain unresolved until those sites are running: {err}" + )); + let (preview_peer_identities, preview_peer_ports) = + preview_placeholder_peer_mesh_state(&mesh_plan); + materialize_direct_runtime_with_existing( + &plan_root, + runtime_root, + &direct_plan, + &mesh_plan, + router_mesh_port, + DirectExistingMeshState { + reuse_existing: false, + peer_ports_by_id: &preview_peer_ports, + peer_identities_by_id: &preview_peer_identities, + }, + )? + } + Err(err) => return Err(err), + }; let router_binary = resolve_runtime_binary("amber-router")?; let mut processes = Vec::new(); let mut router_public_key_b64 = None; if let Some(router) = direct_plan.router.as_ref() { - let router_config = read_mesh_config_public(&runtime_root.join(&router.mesh_config_path))?; - router_public_key_b64 = Some( - base64::engine::general_purpose::STANDARD.encode(router_config.identity.public_key), - ); + let router_config_path = runtime_root.join(&router.mesh_config_path); + match read_mesh_config_public(&router_config_path) { + Ok(router_config) => { + router_public_key_b64 = Some( + base64::engine::general_purpose::STANDARD + .encode(router_config.identity.public_key), + ); + } + Err(err) => inspectability_warnings.push(format!( + "failed to inspect direct router mesh config {}: {err}", + router_config_path.display() + )), + } let paths = DirectControlSocketPaths { artifact_link: resolve_direct_artifact_path(&plan_root, &router.control_socket_path), current_link: direct_current_control_socket_path(&plan_root), @@ -302,9 +339,56 @@ pub(crate) fn build_direct_site_launch_preview( Ok(DirectSiteLaunchPreview { router_public_key_b64, processes, + inspectability_warnings, }) } +fn missing_existing_peer_identity(err: &miette::Report) -> bool { + err.to_string() + .contains("mesh provision plan requires existing peer identity") +} + +fn preview_placeholder_peer_mesh_state( + mesh_plan: &MeshProvisionPlan, +) -> (BTreeMap, BTreeMap) { + let target_ids = mesh_plan + .targets + .iter() + .map(|target| target.config.identity.id.as_str()) + .collect::>(); + let mesh_scope = mesh_plan + .targets + .first() + .and_then(|target| target.config.identity.mesh_scope.clone()); + let peer_ids = mesh_plan + .targets + .iter() + .flat_map(|target| target.config.peers.iter()) + .filter(|peer| !target_ids.contains(peer.id.as_str())) + .map(|peer| peer.id.clone()) + .collect::>(); + let peer_identities = peer_ids + .iter() + .map(|peer| { + let identity = MeshIdentity::generate(peer.clone(), mesh_scope.clone()); + ( + peer.clone(), + MeshIdentityPublic { + id: identity.id, + public_key: identity.public_key, + mesh_scope: identity.mesh_scope, + }, + ) + }) + .collect(); + let peer_ports = peer_ids + .into_iter() + .enumerate() + .map(|(index, peer_id)| (peer_id, 39_000 + index as u16)) + .collect(); + (peer_identities, peer_ports) +} + pub(crate) fn load_direct_runtime_inputs(plan_path: &Path) -> Result { let plan_root = plan_path .parent() diff --git a/cli/src/direct_runtime/runtime.rs b/cli/src/direct_runtime/runtime.rs index 735adcae..6ea1e18f 100644 --- a/cli/src/direct_runtime/runtime.rs +++ b/cli/src/direct_runtime/runtime.rs @@ -111,6 +111,8 @@ pub(crate) struct DirectSiteLaunchPreview { pub(crate) router_public_key_b64: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub(crate) processes: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(crate) inspectability_warnings: Vec, } #[derive(Debug)] @@ -184,10 +186,12 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { args.existing_peer_ports.as_deref(), "direct existing peer ports", )?; - let existing_peer_identities_by_id = read_existing_peer_identities( + let discovered_peer_identities_by_id = read_existing_peer_identities( args.existing_peer_identities.as_deref(), "direct existing peer identities", )?; + let existing_peer_identities_by_id = + merge_existing_mesh_peer_identities(&mesh_plan, &discovered_peer_identities_by_id)?; let supervision = async { let mut sandbox = DirectSandbox::detect(&runtime_root); @@ -405,6 +409,33 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { work_dir.display() ) })?; + let control_socket_path = direct_component_control_socket_path(&work_dir, component.id); + let control_socket_dir = control_socket_path + .parent() + .ok_or_else(|| miette::miette!("invalid sidecar control socket path"))? + .to_path_buf(); + fs::create_dir_all(&control_socket_dir) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to create sidecar control directory {}", + control_socket_dir.display() + ) + })?; + if control_socket_path.exists() { + fs::remove_file(&control_socket_path) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to remove stale sidecar control socket {}", + control_socket_path.display() + ) + })?; + } + env.insert( + "AMBER_ROUTER_CONTROL_SOCKET_PATH".to_string(), + control_socket_path.display().to_string(), + ); let spec = ProcessSpec { name: component.sidecar.log_name.clone(), program: router_binary.clone(), @@ -419,7 +450,7 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { dest: runtime_root.join("mesh"), }], writable_dirs: Vec::new(), - bind_dirs: Vec::new(), + bind_dirs: vec![control_socket_dir], bind_mounts: Vec::new(), hidden_paths: Vec::new(), network: { @@ -572,6 +603,36 @@ pub(crate) fn read_existing_peer_identities( Ok(identities) } +fn merge_existing_mesh_peer_identities( + mesh_plan: &MeshProvisionPlan, + discovered_peer_identities_by_id: &BTreeMap, +) -> Result> { + let mut merged = mesh_plan + .existing_peer_identities + .iter() + .cloned() + .map(|identity| (identity.id.clone(), identity)) + .collect::>(); + + for (peer_id, discovered_identity) in discovered_peer_identities_by_id { + if let Some(embedded_identity) = merged.get(peer_id) { + if embedded_identity.id != discovered_identity.id + || embedded_identity.public_key != discovered_identity.public_key + || embedded_identity.mesh_scope != discovered_identity.mesh_scope + { + return Err(miette::miette!( + "existing peer identity `{peer_id}` from the direct runtime arguments does \ + not match the embedded mesh provision plan" + )); + } + continue; + } + merged.insert(peer_id.clone(), discovered_identity.clone()); + } + + Ok(merged) +} + pub(crate) fn project_existing_direct_peer_identities( runtime_root: &Path, direct_plan: &DirectPlan, @@ -684,6 +745,14 @@ pub(crate) fn direct_runtime_control_socket_path(runtime_root: &Path) -> PathBuf amber_mesh::stable_temp_socket_path("amber-direct-control", "runtime", runtime_root) } +fn direct_component_control_socket_path(work_dir: &Path, component_id: usize) -> PathBuf { + amber_mesh::stable_temp_socket_path( + "amber-direct-control", + &format!("sidecar-{component_id}"), + work_dir, + ) +} + #[cfg(unix)] pub(crate) fn ensure_direct_control_socket_link( link: &Path, @@ -977,13 +1046,14 @@ pub(crate) fn rewrite_direct_mesh_peer_addrs( mesh_port_by_peer_id: &HashMap, ) -> Result<()> { for route in &mut config.outbound { + let addr = route.peer_addr.parse::().map_err(|err| { + miette::miette!("invalid mesh peer address {}: {err}", route.peer_addr) + })?; let port = mesh_port_by_peer_id .get(route.peer_id.as_str()) .copied() + .or_else(|| (addr.port() != 0).then_some(addr.port())) .ok_or_else(|| miette::miette!("missing mesh port for peer {}", route.peer_id))?; - let addr = route.peer_addr.parse::().map_err(|err| { - miette::miette!("invalid mesh peer address {}: {err}", route.peer_addr) - })?; route.peer_addr = SocketAddr::new(addr.ip(), port).to_string(); } @@ -992,13 +1062,14 @@ pub(crate) fn rewrite_direct_mesh_peer_addrs( peer_addr, peer_id, .. } = &mut route.target { + let addr = peer_addr + .parse::() + .map_err(|err| miette::miette!("invalid mesh peer address {}: {err}", peer_addr))?; let port = mesh_port_by_peer_id .get(peer_id.as_str()) .copied() + .or_else(|| (addr.port() != 0).then_some(addr.port())) .ok_or_else(|| miette::miette!("missing mesh port for peer {}", peer_id))?; - let addr = peer_addr - .parse::() - .map_err(|err| miette::miette!("invalid mesh peer address {}: {err}", peer_addr))?; peer_addr.clear(); peer_addr.push_str(&SocketAddr::new(addr.ip(), port).to_string()); } @@ -1091,7 +1162,6 @@ pub(crate) fn materialize_direct_runtime( reuse_existing: bool, ) -> Result { let empty_peer_ports = BTreeMap::new(); - let empty_peer_identities = BTreeMap::new(); materialize_direct_runtime_with_existing( plan_root, runtime_root, @@ -1101,7 +1171,7 @@ pub(crate) fn materialize_direct_runtime( DirectExistingMeshState { reuse_existing, peer_ports_by_id: &empty_peer_ports, - peer_identities_by_id: &empty_peer_identities, + peer_identities_by_id: &BTreeMap::new(), }, ) } @@ -1127,8 +1197,10 @@ pub(crate) fn materialize_direct_runtime_with_existing( if runtime_state_path.exists() { let _ = fs::remove_file(&runtime_state_path); } + let merged_peer_identities = + merge_existing_mesh_peer_identities(mesh_plan, existing.peer_identities_by_id)?; let existing_mesh_peer_identities = - required_existing_mesh_peer_identities(mesh_plan, existing.peer_identities_by_id)?; + required_existing_mesh_peer_identities(mesh_plan, &merged_peer_identities)?; provision_mesh_filesystem_with_peer_identities( mesh_plan, runtime_root, @@ -1262,3 +1334,128 @@ pub(crate) fn write_mesh_config_public(path: &Path, config: &MeshConfigPublic) - .into_diagnostic() .wrap_err_with(|| format!("failed to write mesh config {}", path.display())) } + +#[cfg(test)] +mod tests { + use amber_mesh::{InboundRoute, MeshPeer}; + + use super::*; + + fn mesh_identity_public(id: &str, public_key: [u8; 32]) -> MeshIdentityPublic { + MeshIdentityPublic { + id: id.to_string(), + public_key, + mesh_scope: Some("amber.test".to_string()), + } + } + + #[test] + fn merge_existing_mesh_peer_identities_keeps_embedded_plan_identities() { + let embedded_identity = mesh_identity_public("/site/compose/router", [7; 32]); + let mesh_plan = MeshProvisionPlan { + version: MESH_PROVISION_PLAN_VERSION.to_string(), + identity_seed: None, + existing_peer_identities: vec![embedded_identity.clone()], + targets: Vec::new(), + }; + + let merged = + merge_existing_mesh_peer_identities(&mesh_plan, &BTreeMap::new()).expect("merge"); + + let merged_identity = merged + .get("/site/compose/router") + .expect("embedded identity should remain available"); + assert_eq!(merged_identity.id, embedded_identity.id); + assert_eq!(merged_identity.public_key, embedded_identity.public_key); + assert_eq!(merged_identity.mesh_scope, embedded_identity.mesh_scope); + } + + #[test] + fn merge_existing_mesh_peer_identities_rejects_conflicts() { + let mesh_plan = MeshProvisionPlan { + version: MESH_PROVISION_PLAN_VERSION.to_string(), + identity_seed: None, + existing_peer_identities: vec![mesh_identity_public("/peer", [1; 32])], + targets: Vec::new(), + }; + let discovered = + BTreeMap::from([("/peer".to_string(), mesh_identity_public("/peer", [2; 32]))]); + + let err = merge_existing_mesh_peer_identities(&mesh_plan, &discovered) + .expect_err("conflicting embedded and discovered identities should fail"); + + assert!( + err.to_string() + .contains("does not match the embedded mesh provision plan"), + "unexpected merge error: {err}", + ); + } + + #[test] + fn rewrite_direct_mesh_peer_addrs_preserves_authored_external_peer_ports() { + let mut config = MeshConfigPublic { + identity: mesh_identity_public("/site/direct/router", [3; 32]), + mesh_listen: "127.0.0.1:24000".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: vec![MeshPeer { + id: "/site/compose/router".to_string(), + public_key: [4; 32], + }], + inbound: vec![InboundRoute { + route_id: "in".to_string(), + capability: "site-controller".to_string(), + capability_kind: None, + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: InboundTarget::MeshForward { + peer_addr: "127.0.0.1:32002".to_string(), + peer_id: "/site/compose/router".to_string(), + route_id: "route".to_string(), + capability: "site-controller".to_string(), + }, + allowed_issuers: Vec::new(), + }], + outbound: vec![OutboundRoute { + route_id: "out".to_string(), + slot: "compose".to_string(), + capability_kind: None, + capability_profile: None, + listen_port: 25000, + listen_addr: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + peer_addr: "127.0.0.1:32001".to_string(), + peer_id: "/site/compose/router".to_string(), + capability: "site-controller".to_string(), + }], + transport: amber_mesh::TransportConfig::NoiseIk {}, + }; + + rewrite_direct_mesh_peer_addrs(&mut config, &HashMap::new()).expect("rewrite"); + + assert_eq!(config.outbound[0].peer_addr, "127.0.0.1:32001"); + let InboundTarget::MeshForward { peer_addr, .. } = &config.inbound[0].target else { + panic!("expected mesh forward route"); + }; + assert_eq!(peer_addr, "127.0.0.1:32002"); + } + + #[test] + fn direct_component_control_socket_path_stays_short_on_long_work_dirs() { + let work_dir = Path::new( + "/Users/example/Developer/amber/target/cli-test-outputs/\ + mixed-run-doc-example-detach-very-long/state/runs/run-123/state/direct_local/runtime/\ + work/components/c2-web", + ); + let socket = direct_component_control_socket_path(work_dir, 2); + let rendered = socket.as_os_str().to_string_lossy(); + + assert!( + rendered.len() < 104, + "direct sidecar control socket path must fit within unix socket limits: {rendered}", + ); + } +} diff --git a/cli/src/main.rs b/cli/src/main.rs index 5df985e9..c948de8a 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -877,11 +877,7 @@ async fn main() -> Result<()> { } Command::RunSiteSupervisor(args) => mixed_run::run_site_supervisor(args.plan).await, Command::RunSiteController(args) => { - amber_site_controller::run_site_controller( - args.plan, - mixed_run::site_controller_runtime(), - ) - .await + amber_site_controller::run_site_controller_default(args.plan).await } Command::RunDetachedCoordinator(args) => run_detached_coordinator(args).await, Command::RunObservabilitySink(args) => { diff --git a/cli/src/mixed_run.rs b/cli/src/mixed_run.rs index 465580c5..4bf93de0 100644 --- a/cli/src/mixed_run.rs +++ b/cli/src/mixed_run.rs @@ -1,8 +1,6 @@ use std::{ collections::{BTreeMap, BTreeSet, HashMap}, - env, - fmt::Write as _, - fs, + env, fs, future::Future, io::{Read as _, Write as _}, net::{SocketAddr, TcpListener, TcpStream}, @@ -17,70 +15,39 @@ use std::{ use amber_compiler::{ mesh::ProxyMetadata, - reporter::{ - direct::{DirectPlan, DirectRuntimeUrlSource}, - vm::{VM_PLAN_FILENAME, VmPlan}, - }, + reporter::vm::{VM_PLAN_FILENAME, VmPlan}, run_plan::{RunLink, RunPlan, RunSitePlan, SiteKind}, }; use amber_manifest::{CapabilityKind, CapabilityTransport, NetworkProtocol}; use amber_mesh::{ - InboundRoute, InboundTarget, MESH_CONFIG_FILENAME, MESH_IDENTITY_FILENAME, MeshConfig, - MeshConfigPublic, MeshIdentity, MeshIdentityPublic, MeshIdentitySecret, MeshPeer, MeshProtocol, - MeshProvisionOutput, MeshProvisionPlan, MeshProvisionTargetKind, OutboundRoute, - TransportConfig, component_route_id, http_route_plugins_for_capability_kind, - router_dynamic_export_route_id, router_export_route_id, + InboundRoute, InboundTarget, MeshConfig, MeshConfigPublic, MeshIdentity, MeshIdentityPublic, + MeshPeer, MeshProtocol, OutboundRoute, TransportConfig, component_route_id, + router_export_route_id, telemetry::{SCENARIO_RUN_ID_ENV, SCENARIO_SCOPE_ENV}, }; use amber_proxy::{ - ControlEndpoint, RouterDiscovery, apply_route_overlay_with_retry, discover_router_for_output, - fetch_router_identity, load_output_proxy_metadata, register_export_peer_with_retry, - register_external_slot_with_retry, revoke_route_overlay_with_retry, + ControlEndpoint, RouterDiscovery, discover_router_for_output, fetch_router_identity, + load_output_proxy_metadata, register_export_peer_with_retry, register_external_slot_with_retry, }; use amber_site_controller::{ - DesiredExportPeerOverlay as ControllerDesiredExportPeerOverlay, - DesiredExternalSlotOverlay as ControllerDesiredExternalSlotOverlay, DynamicInputRouteRecord, - DynamicInputRouteTarget, DynamicProxyExportRecord, DynamicSitePlanRecord, - LaunchedSite as ControllerLaunchedSite, - LiveComponentRuntimeMetadata as ControllerLiveComponentRuntimeMetadata, - SharedSiteControllerRuntime, SiteControllerPlan, - SiteControllerRuntime as FrameworkSiteControllerRuntime, SiteControllerRuntimeFuture, - SiteControllerRuntimePlan as ControllerSiteControllerRuntimePlan, - SiteReceipt as ControllerSiteReceipt, + assign_compose_egress_network_subnets, cleanup_dynamic_site_children, + observability_endpoint_for_site, prepare_kubernetes_artifact_namespace, walk_files, }; use base64::Engine as _; use miette::{Context as _, IntoDiagnostic as _, Result}; -use opentelemetry_proto::tonic::{ - collector::logs::v1::ExportLogsServiceRequest, - common::v1::{AnyValue as OtlpAnyValue, KeyValue as OtlpKeyValue, any_value}, - logs::v1::LogRecord as OtlpLogRecord, -}; -use prost::Message as _; use serde::{Deserialize, Serialize}; -use serde_json::{Map as JsonMap, Value as JsonValue}; -use sha2::{Digest as _, Sha256}; -use tokio::{ - io::{AsyncReadExt as _, AsyncWriteExt as _}, - sync::Mutex as AsyncMutex, - time::{Instant, sleep}, -}; +use tokio::time::{Instant, sleep}; use url::Url; use crate::{ DirectLaunchProcessPreview, DirectSiteLaunchPreview, build_direct_site_launch_preview, - direct_current_control_socket_path, - direct_runtime::ensure_direct_control_socket_link, - direct_runtime_state_path, + direct_current_control_socket_path, direct_runtime_state_path, run_inputs::{collect_run_interface, validate_export_bindings, validate_slot_bindings}, site_proxy_metadata::load_site_proxy_metadata, - tcp_readiness::{ - endpoint_accepts_stable_connection, endpoint_returns_http_response, wait_for_http_response, - wait_for_stable_endpoint, - }, + tcp_readiness::{wait_for_http_response, wait_for_stable_endpoint}, vm_runtime::{ TCG_VM_STARTUP_TIMEOUT, VmLaunchPreview, VmRuntimeState, VmSiteLaunchPreview, - build_vm_site_launch_preview, ensure_control_socket_link, vm_current_control_socket_path, - vm_endpoint_forward_ready_timeout, vm_uses_tcg_accel, write_vm_runtime_state, + build_vm_site_launch_preview, vm_current_control_socket_path, vm_uses_tcg_accel, }, }; mod launch_bundle; @@ -92,29 +59,11 @@ pub(crate) use self::{launch_bundle::*, outside_proxy::*, supervisor::*}; const RECEIPT_SCHEMA: &str = "amber.run.receipt"; const RECEIPT_VERSION: u32 = 3; const LAUNCH_BUNDLE_SCHEMA: &str = "amber.run.launch_bundle"; -const DYNAMIC_COMPOSE_CHILD_SCHEMA: &str = "amber.run.dynamic_compose_child"; -const DYNAMIC_COMPOSE_CHILD_VERSION: u32 = 1; -const DYNAMIC_COMPOSE_MESH_ROOT: &str = ".amber/mesh"; -const DYNAMIC_ROUTE_OVERLAY_FILENAME: &str = "site-router-overlay.json"; -const DYNAMIC_PROXY_EXPORTS_FILENAME: &str = "proxy-exports.json"; -const COMPOSE_PROVISIONER_SERVICE_NAME: &str = "amber-provisioner"; -const COMPOSE_ROUTER_SERVICE_NAME: &str = "amber-router"; -const KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH: &str = "01-configmaps/amber-mesh-provision.yaml"; -const KUBERNETES_PROVISIONER_JOB_PATH: &str = "02-rbac/amber-provisioner-job.yaml"; -const KUBERNETES_PROVISIONER_ROLE_PATH: &str = "02-rbac/amber-provisioner-role.yaml"; -const KUBERNETES_PROVISIONER_ROLEBINDING_PATH: &str = "02-rbac/amber-provisioner-rolebinding.yaml"; -const KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH: &str = "02-rbac/amber-provisioner-sa.yaml"; -const KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME: &str = "amber-router-external"; -const KUBERNETES_ROUTER_COMPONENT_NAME: &str = "amber-router"; const LAUNCH_BUNDLE_VERSION: u32 = 1; const SITE_STATE_SCHEMA: &str = "amber.run.site_state"; const SITE_STATE_VERSION: u32 = 2; const SITE_PLAN_SCHEMA: &str = "amber.run.site_supervisor_plan"; const SITE_PLAN_VERSION: u32 = 1; -const SITE_CONTROLLER_RUNTIME_PLAN_SCHEMA: &str = "amber.run.site_controller_runtime_plan"; -const SITE_CONTROLLER_RUNTIME_PLAN_VERSION: u32 = 1; -const SITE_CONTROLLER_RUNTIME_STATE_SCHEMA: &str = "amber.run.site_controller_runtime_state"; -const SITE_CONTROLLER_RUNTIME_STATE_VERSION: u32 = 1; const DESIRED_LINKS_SCHEMA: &str = "amber.run.desired_links"; const DESIRED_LINKS_VERSION: u32 = 1; const OTLP_SINK_PLAN_SCHEMA: &str = "amber.run.observability_sink"; @@ -125,8 +74,6 @@ const OUTSIDE_PROXY_PLAN_SCHEMA: &str = "amber.run.outside_proxy"; const OUTSIDE_PROXY_PLAN_VERSION: u32 = 1; const OUTSIDE_PROXY_STATE_SCHEMA: &str = "amber.run.outside_proxy_state"; const OUTSIDE_PROXY_STATE_VERSION: u32 = 1; -const PERSISTED_TRACE_EVENT_SCHEMA: &str = "amber.trace.event"; -const PERSISTED_TRACE_EVENT_VERSION: u32 = 1; const ROUTER_CONTROL_TIMEOUT: Duration = Duration::from_secs(30); const SUPERVISOR_POLL_INTERVAL: Duration = Duration::from_millis(500); @@ -220,231 +167,6 @@ fn workspace_executable_from( )) } -#[derive(Clone, Default)] -pub(crate) struct CliSiteControllerRuntime { - bridge_proxies: Arc>>, -} - -pub(crate) fn site_controller_runtime() -> SharedSiteControllerRuntime { - Arc::new(CliSiteControllerRuntime::default()) -} - -fn local_site_receipt(receipt: &ControllerSiteReceipt) -> SiteReceipt { - SiteReceipt { - kind: receipt.kind, - artifact_dir: receipt.artifact_dir.clone(), - supervisor_pid: receipt.supervisor_pid, - process_pid: receipt.process_pid, - compose_project: receipt.compose_project.clone(), - kubernetes_namespace: receipt.kubernetes_namespace.clone(), - port_forward_pid: receipt.port_forward_pid, - context: receipt.context.clone(), - router_control: receipt.router_control.clone(), - router_mesh_addr: receipt.router_mesh_addr.clone(), - router_identity_id: receipt.router_identity_id.clone(), - router_public_key_b64: receipt.router_public_key_b64.clone(), - site_controller_pid: receipt.site_controller_pid, - site_controller_url: receipt.site_controller_url.clone(), - } -} - -fn local_site_controller_runtime_plan( - plan: &ControllerSiteControllerRuntimePlan, -) -> SiteControllerRuntimePlan { - SiteControllerRuntimePlan { - schema: plan.schema.clone(), - version: plan.version, - run_id: plan.run_id.clone(), - mesh_scope: plan.mesh_scope.clone(), - run_root: plan.run_root.clone(), - site_id: plan.site_id.clone(), - kind: plan.kind, - router_identity_id: plan.router_identity_id.clone(), - artifact_dir: plan.artifact_dir.clone(), - site_state_root: plan.site_state_root.clone(), - listen_addr: plan.listen_addr, - storage_root: plan.storage_root.clone(), - runtime_root: plan.runtime_root.clone(), - router_mesh_port: plan.router_mesh_port, - compose_project: plan.compose_project.clone(), - kubernetes_namespace: plan.kubernetes_namespace.clone(), - context: plan.context.clone(), - observability_endpoint: plan.observability_endpoint.clone(), - launch_env: plan.launch_env.clone(), - } -} - -fn local_launched_site(site: &ControllerLaunchedSite) -> LaunchedSite { - LaunchedSite { - receipt: local_site_receipt(&site.receipt), - router_control: site.router_control.clone(), - router_identity: site.router_identity.clone(), - router_addr: site.router_addr, - } -} - -impl FrameworkSiteControllerRuntime for CliSiteControllerRuntime { - fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()> { - Box::pin(async move { - let mut bridge_proxies = { - let mut guard = self.bridge_proxies.lock().await; - std::mem::take(&mut *guard) - }; - stop_bridge_proxies(&mut bridge_proxies).await - }) - } - - fn resolve_link_external_url<'a>( - &'a self, - provider: &'a ControllerLaunchedSite, - provider_output_dir: &'a Path, - link: &'a RunLink, - consumer_kind: SiteKind, - run_root: &'a Path, - ) -> SiteControllerRuntimeFuture<'a, String> { - Box::pin(async move { - let provider = local_launched_site(provider); - let mut bridge_proxies = self.bridge_proxies.lock().await; - resolve_link_external_url_for_output( - &provider, - provider_output_dir, - link, - consumer_kind, - run_root, - &mut bridge_proxies, - ) - .await - }) - } - - fn prepare_child<'a>( - &'a self, - plan: &'a SiteControllerPlan, - child_id: u64, - site_plan: DynamicSitePlanRecord, - ) -> SiteControllerRuntimeFuture<'a, ()> { - Box::pin(async move { site_controller_prepare_child(plan, child_id, site_plan).await }) - } - - fn publish_child<'a>( - &'a self, - plan: &'a SiteControllerPlan, - child_id: u64, - site_plan: DynamicSitePlanRecord, - ) -> SiteControllerRuntimeFuture<'a, ()> { - Box::pin(async move { site_controller_publish_child(plan, child_id, site_plan).await }) - } - - fn rollback_child<'a>( - &'a self, - plan: &'a SiteControllerPlan, - child_id: u64, - ) -> SiteControllerRuntimeFuture<'a, ()> { - Box::pin(async move { site_controller_rollback_child(plan, child_id).await }) - } - - fn destroy_child<'a>( - &'a self, - plan: &'a SiteControllerPlan, - child_id: u64, - desired_site_plan: Option, - ) -> SiteControllerRuntimeFuture<'a, ()> { - Box::pin( - async move { site_controller_destroy_child(plan, child_id, desired_site_plan).await }, - ) - } - - fn collect_live_component_runtime_metadata( - &self, - plan: &ControllerSiteControllerRuntimePlan, - ) -> Result> { - collect_live_component_runtime_metadata(&local_site_controller_runtime_plan(plan)).map( - |components| { - components - .into_iter() - .map(|(component_id, metadata)| { - ( - component_id, - ControllerLiveComponentRuntimeMetadata { - moniker: metadata.moniker, - host_mesh_addr: metadata.host_mesh_addr, - mesh_config: metadata.mesh_config, - }, - ) - }) - .collect() - }, - ) - } - - fn load_live_site_router_mesh_config( - &self, - plan: &ControllerSiteControllerRuntimePlan, - ) -> Result { - load_live_site_router_mesh_config(&local_site_controller_runtime_plan(plan)) - } - - fn router_mesh_addr_for_consumer( - &self, - provider_kind: SiteKind, - consumer_kind: SiteKind, - router_mesh_addr: &str, - ) -> Result { - router_mesh_addr_for_consumer(provider_kind, consumer_kind, router_mesh_addr) - } - - fn update_desired_overlay_for_consumer( - &self, - site_state_root: &Path, - overlay_id: &str, - overlay: ControllerDesiredExternalSlotOverlay, - ) -> Result<()> { - update_desired_overlay_for_consumer( - site_state_root, - overlay_id, - DesiredExternalSlotOverlay { - slot_name: overlay.slot_name, - url: overlay.url, - }, - ) - } - - fn update_desired_overlay_for_provider( - &self, - site_state_root: &Path, - overlay_id: &str, - overlay: ControllerDesiredExportPeerOverlay, - ) -> Result<()> { - update_desired_overlay_for_provider( - site_state_root, - overlay_id, - DesiredExportPeerOverlay { - export_name: overlay.export_name, - peer_id: overlay.peer_id, - peer_key_b64: overlay.peer_key_b64, - protocol: overlay.protocol, - route_id: overlay.route_id, - }, - ) - } - - fn clear_desired_overlay_for_consumer( - &self, - site_state_root: &Path, - overlay_id: &str, - ) -> Result<()> { - clear_desired_overlay_for_consumer(site_state_root, overlay_id) - } - - fn clear_desired_overlay_for_provider( - &self, - site_state_root: &Path, - overlay_id: &str, - ) -> Result<()> { - clear_desired_overlay_for_provider(site_state_root, overlay_id) - } -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct RunReceipt { pub(crate) schema: String, @@ -497,8 +219,6 @@ struct LaunchBundleObservability { state_path: Option, #[serde(default, skip_serializing_if = "Option::is_none")] requests_log: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - events_ndjson: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] launch_commands: Vec, } @@ -574,23 +294,6 @@ pub(crate) struct ObservabilityReceipt { pub(crate) sink_pid: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) requests_log: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) events_ndjson: Option, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct PersistedTraceEvent { - pub(crate) schema: String, - pub(crate) version: u32, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) observed_at_unix_nano: Option, - pub(crate) message: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) severity: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) trace_id: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub(crate) attributes: BTreeMap, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -702,94 +405,10 @@ pub(crate) struct SiteSupervisorPlan { site_controller_plan_path: Option, #[serde(default, skip_serializing_if = "Option::is_none")] site_controller_url: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - launch_env: BTreeMap, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct SiteControllerRuntimePlan { - pub(crate) schema: String, - pub(crate) version: u32, - pub(crate) run_id: String, - pub(crate) mesh_scope: String, - pub(crate) run_root: String, - pub(crate) site_id: String, - pub(crate) kind: SiteKind, - pub(crate) router_identity_id: String, - pub(crate) artifact_dir: String, - pub(crate) site_state_root: String, - pub(crate) listen_addr: SocketAddr, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) storage_root: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) runtime_root: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) router_mesh_port: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) compose_project: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) kubernetes_namespace: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) context: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) observability_endpoint: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub(crate) launch_env: BTreeMap, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -struct SiteControllerRuntimeState { - schema: String, - version: u32, - run_id: String, - site_id: String, - kind: SiteKind, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - children: BTreeMap, -} - -#[derive(Clone, Debug)] -pub(crate) struct LiveComponentRuntimeMetadata { - pub(crate) moniker: String, - pub(crate) host_mesh_addr: String, - pub(crate) mesh_config: MeshConfigPublic, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -struct SiteControllerRuntimeChildRecord { - child_id: u64, - artifact_root: String, #[serde(default, skip_serializing_if = "Vec::is_empty")] - assigned_components: Vec, + controller_route_ports: Vec, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - proxy_exports: BTreeMap, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - routed_inputs: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - process_pid: Option, - published: bool, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -struct DynamicComposeChildMetadata { - schema: String, - version: u32, - services: Vec, - readiness_services: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -struct StoredRouteOverlayPayload { - #[serde(default, skip_serializing_if = "Vec::is_empty")] - peers: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - inbound_routes: Vec, -} - -#[derive(Debug, Deserialize)] -struct KubernetesSecretPayload { - #[serde(default)] - data: BTreeMap, + launch_env: BTreeMap, } struct SupervisorPlanInput<'a> { @@ -816,6 +435,10 @@ pub(crate) struct MaterializedSite { pub(crate) site_plan: RunSitePlan, pub(crate) artifact_dir: PathBuf, pub(crate) site_state_root: PathBuf, + pub(crate) controller_state_path: PathBuf, + pub(crate) controller_plan_path: PathBuf, + pub(crate) controller_listen_addr: SocketAddr, + pub(crate) controller_url: String, pub(crate) base_supervisor_plan: SiteSupervisorPlan, } @@ -892,7 +515,6 @@ struct ObservabilitySinkPlan { listen_addr: String, advertise_endpoint: String, requests_log: String, - events_ndjson: String, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -926,10 +548,6 @@ struct RunOutsideProxyContext { #[derive(Clone, Debug)] struct RunOutsideExport { site_id: String, - component: String, - provide: String, - capability_kind: Option, - capability_profile: Option, protocol: String, } @@ -980,16 +598,6 @@ pub(crate) struct BridgeProxyKey { pub(crate) consumer_kind: SiteKind, } -pub(crate) async fn stop_bridge_proxies( - bridge_proxies: &mut BTreeMap, -) -> Result<()> { - for bridge in bridge_proxies.values_mut() { - supervisor::stop_child(&mut bridge.child).await?; - } - bridge_proxies.clear(); - Ok(()) -} - #[derive(Debug)] struct SupervisorRuntime { site_process: Option, @@ -1399,7 +1007,6 @@ pub(crate) async fn run_observability_sink(plan_path: PathBuf) -> Result<()> { endpoint: plan.advertise_endpoint.clone(), sink_pid: Some(std::process::id()), requests_log: Some(plan.requests_log.clone()), - events_ndjson: Some(plan.events_ndjson.clone()), }, )?; @@ -1409,11 +1016,9 @@ pub(crate) async fn run_observability_sink(plan_path: PathBuf) -> Result<()> { } match listener.accept() { - Ok((mut stream, _)) => handle_otlp_connection( - &mut stream, - Path::new(&plan.requests_log), - Path::new(&plan.events_ndjson), - )?, + Ok((mut stream, _)) => { + handle_otlp_connection(&mut stream, Path::new(&plan.requests_log))? + } Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => { std::thread::sleep(Duration::from_millis(100)); } @@ -1424,5827 +1029,126 @@ pub(crate) async fn run_observability_sink(plan_path: PathBuf) -> Result<()> { } } -fn build_site_controller_runtime_app( - plan: SiteControllerRuntimePlan, -) -> Result { - let state_path = site_controller_runtime_state_path(Path::new(&plan.site_state_root)); - let initial_state = if state_path.is_file() { - read_json(&state_path, "site controller runtime state")? - } else { - let state = SiteControllerRuntimeState { - schema: SITE_CONTROLLER_RUNTIME_STATE_SCHEMA.to_string(), - version: SITE_CONTROLLER_RUNTIME_STATE_VERSION, - run_id: plan.run_id.clone(), - site_id: plan.site_id.clone(), - kind: plan.kind, - children: BTreeMap::new(), - }; - write_json(&state_path, &state)?; - state - }; - Ok(SiteControllerRuntimeApp { - plan, - state_path, - state: Arc::new(AsyncMutex::new(initial_state)), - }) -} - -pub(crate) fn site_controller_runtime_plan_from_controller_plan( - plan: &SiteControllerPlan, -) -> SiteControllerRuntimePlan { - SiteControllerRuntimePlan { - schema: SITE_CONTROLLER_RUNTIME_PLAN_SCHEMA.to_string(), - version: SITE_CONTROLLER_RUNTIME_PLAN_VERSION, - run_id: plan.run_id.clone(), - mesh_scope: plan.mesh_scope.clone(), - run_root: plan.run_root.clone(), - site_id: plan.site_id.clone(), - kind: plan.kind, - router_identity_id: plan.router_identity_id.clone(), - artifact_dir: plan.artifact_dir.clone(), - site_state_root: plan.site_state_root.clone(), - listen_addr: plan.listen_addr, - storage_root: plan.storage_root.clone(), - runtime_root: plan.runtime_root.clone(), - router_mesh_port: plan.router_mesh_port, - compose_project: plan.compose_project.clone(), - kubernetes_namespace: plan.kubernetes_namespace.clone(), - context: plan.context.clone(), - observability_endpoint: plan.observability_endpoint.clone(), - launch_env: plan.launch_env.clone(), - } +pub(crate) fn reserve_loopback_port() -> Result { + let listener = TcpListener::bind(("127.0.0.1", 0)) + .into_diagnostic() + .wrap_err("failed to allocate a loopback port")?; + Ok(listener.local_addr().into_diagnostic()?.port()) } -pub(crate) async fn site_controller_prepare_child( - plan: &SiteControllerPlan, - child_id: u64, - site_plan: DynamicSitePlanRecord, -) -> Result<()> { - let app = - build_site_controller_runtime_app(site_controller_runtime_plan_from_controller_plan(plan))?; - site_controller_runtime_prepare_child(&app, child_id, site_plan).await +pub(crate) fn new_run_id() -> String { + let millis = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + format!("run-{millis:x}-{:x}", std::process::id()) } -pub(crate) async fn site_controller_publish_child( - plan: &SiteControllerPlan, - child_id: u64, - site_plan: DynamicSitePlanRecord, -) -> Result<()> { - let app = - build_site_controller_runtime_app(site_controller_runtime_plan_from_controller_plan(plan))?; - site_controller_runtime_publish_child(&app, child_id, site_plan).await +fn receipt_path(run_root: &Path) -> PathBuf { + run_root.join("receipt.json") } -pub(crate) async fn site_controller_rollback_child( - plan: &SiteControllerPlan, - child_id: u64, -) -> Result<()> { - let app = - build_site_controller_runtime_app(site_controller_runtime_plan_from_controller_plan(plan))?; - site_controller_runtime_rollback_child(&app, child_id).await +fn run_plan_path(run_root: &Path) -> PathBuf { + run_root.join("run-plan.json") } -pub(crate) async fn site_controller_destroy_child( - plan: &SiteControllerPlan, - child_id: u64, - desired_site_plan: Option, -) -> Result<()> { - let app = - build_site_controller_runtime_app(site_controller_runtime_plan_from_controller_plan(plan))?; - site_controller_runtime_destroy_child(&app, child_id, desired_site_plan).await +fn launch_bundle_manifest_path(run_root: &Path) -> PathBuf { + run_root.join("launch-bundle.json") } -#[derive(Clone)] -struct SiteControllerRuntimeApp { - plan: SiteControllerRuntimePlan, - state_path: PathBuf, - state: Arc>, +pub(crate) fn site_state_path(state_root: &Path, site_id: &str) -> PathBuf { + state_root.join(site_id).join("manager-state.json") } -fn site_controller_runtime_state_path(site_state_root: &Path) -> PathBuf { - site_state_root.join("site-controller-runtime-state.json") +fn state_path_for_plan(plan: &SiteSupervisorPlan) -> PathBuf { + Path::new(&plan.site_state_root).join("manager-state.json") } -fn site_controller_runtime_child_root(plan: &SiteControllerRuntimePlan, child_id: u64) -> PathBuf { - Path::new(&plan.site_state_root) - .join("framework-component") - .join("children") - .join(child_id.to_string()) +fn site_supervisor_plan_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("site-supervisor-plan.json") } -pub(super) fn site_controller_runtime_child_root_for_site( - site_state_root: &Path, - child_id: u64, -) -> PathBuf { - site_state_root - .join("framework-component") - .join("children") - .join(child_id.to_string()) +pub(crate) fn site_controller_plan_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("site-controller-plan.json") } -fn site_controller_runtime_child_artifact_root( - plan: &SiteControllerRuntimePlan, - child_id: u64, -) -> PathBuf { - site_controller_runtime_child_root(plan, child_id).join("artifact") +pub(crate) fn desired_links_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("desired-links.json") } -fn site_controller_runtime_child_runtime_root( - plan: &SiteControllerRuntimePlan, - child_id: u64, -) -> PathBuf { - site_controller_runtime_child_root(plan, child_id).join("runtime") +fn commit_marker_path(run_root: &Path) -> PathBuf { + run_root.join("committed") } -fn site_controller_runtime_child_storage_root( - plan: &SiteControllerRuntimePlan, - child_id: u64, -) -> PathBuf { - site_controller_runtime_child_root(plan, child_id).join("storage") +fn coordinator_lock_path(run_root: &Path) -> PathBuf { + run_root.join("coordinator.lock") } -fn site_controller_runtime_child_peer_ports_path( - plan: &SiteControllerRuntimePlan, - child_id: u64, -) -> PathBuf { - site_controller_runtime_child_root(plan, child_id).join("existing-peer-ports.json") +fn stop_marker_path(run_root: &Path) -> PathBuf { + run_root.join("stop-requested") } -fn site_controller_runtime_child_peer_identities_path( - plan: &SiteControllerRuntimePlan, - child_id: u64, -) -> PathBuf { - site_controller_runtime_child_root(plan, child_id).join("existing-peer-identities.json") +async fn wait_for_stop_request(stop_requested: &AtomicBool, run_root: &Path) { + loop { + if stop_requested.load(Ordering::SeqCst) || stop_marker_path(run_root).exists() { + return; + } + sleep(Duration::from_millis(100)).await; + } } -fn dynamic_compose_child_metadata_path(artifact_root: &Path) -> PathBuf { - artifact_root.join(".amber").join("compose-child.json") +async fn run_until_stop( + run_root: &Path, + stop_requested: &AtomicBool, + future: F, +) -> Result> +where + F: Future>, +{ + tokio::pin!(future); + tokio::select! { + result = &mut future => result.map(Some), + _ = wait_for_stop_request(stop_requested, run_root) => Ok(None), + } } -fn dynamic_route_overlay_path(artifact_root: &Path) -> PathBuf { - artifact_root - .join(".amber") - .join(DYNAMIC_ROUTE_OVERLAY_FILENAME) +async fn router_mesh_listener_ready(addr: SocketAddr) -> bool { + matches!( + tokio::time::timeout( + Duration::from_millis(250), + tokio::net::TcpStream::connect(addr) + ) + .await, + Ok(Ok(_)) + ) } -fn write_dynamic_route_overlay_payload( - artifact_root: &Path, - payload: &StoredRouteOverlayPayload, -) -> Result<()> { - let path = dynamic_route_overlay_path(artifact_root); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - write_json(&path, payload) +fn observability_plan_path(run_root: &Path) -> PathBuf { + run_root.join("observability").join("sink-plan.json") } -fn dynamic_proxy_exports_path(artifact_root: &Path) -> PathBuf { - artifact_root - .join(".amber") - .join(DYNAMIC_PROXY_EXPORTS_FILENAME) +fn observability_state_path(run_root: &Path) -> PathBuf { + run_root.join("observability").join("sink-state.json") } -fn write_dynamic_proxy_exports_metadata( - artifact_root: &Path, - proxy_exports: &BTreeMap, -) -> Result<()> { - if proxy_exports.is_empty() { - return Ok(()); - } - let path = dynamic_proxy_exports_path(artifact_root); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - write_json(&path, proxy_exports) +fn outside_proxy_plan_path(run_root: &Path) -> PathBuf { + run_root.join("outside-proxy-plan.json") } -fn load_dynamic_proxy_exports_metadata( - artifact_root: &Path, -) -> Result> { - let path = dynamic_proxy_exports_path(artifact_root); - if !path.is_file() { - return Ok(BTreeMap::new()); - } - read_json(&path, "dynamic proxy exports") +fn outside_proxy_state_path(run_root: &Path) -> PathBuf { + run_root.join("outside-proxy-state.json") } -pub(super) fn cleanup_dynamic_site_children(site_state_root: &Path, kind: SiteKind) -> Result<()> { - let state_path = site_controller_runtime_state_path(site_state_root); - if !state_path.is_file() { - return Ok(()); - } - let mut state: SiteControllerRuntimeState = - read_json(&state_path, "site controller runtime state")?; - let compose_supervisor_plan = matches!(kind, SiteKind::Compose) - .then(|| { - read_json::( - &site_supervisor_plan_path(site_state_root), - "site supervisor plan", - ) - }) - .transpose()?; - for child in state.children.values() { - if let Some(plan) = compose_supervisor_plan.as_ref() { - cleanup_dynamic_compose_child(plan, child)?; - } - if let Some(pid) = child.process_pid { - terminate_pid(pid, site_ready_timeout_for_kind(kind))?; - } - remove_dir_if_exists(&site_controller_runtime_child_root_for_site( - site_state_root, - child.child_id, - ))?; - } - if state.children.is_empty() { - return Ok(()); - } - state.children.clear(); - write_json(&state_path, &state) +fn write_commit_marker(run_root: &Path) -> Result<()> { + fs::write(commit_marker_path(run_root), b"committed") + .into_diagnostic() + .wrap_err("failed to write commit marker") } -fn cleanup_dynamic_compose_child( - plan: &SiteSupervisorPlan, - child: &SiteControllerRuntimeChildRecord, -) -> Result<()> { - let artifact_root = Path::new(&child.artifact_root); - if !dynamic_compose_child_metadata_path(artifact_root).is_file() { - return Ok(()); - } - let compose_project = plan.compose_project.as_deref().ok_or_else(|| { - miette::miette!( - "compose site `{}` is missing its compose project name", - plan.site_id - ) - })?; - let status = compose_command(Some(compose_project), artifact_root) - .envs(plan.launch_env.clone()) - .arg("down") - .arg("-v") - .status() +fn write_stop_marker(run_root: &Path) -> Result<()> { + fs::write(stop_marker_path(run_root), b"stop") .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to clean up dynamic compose child for site `{}`", - plan.site_id - ) - })?; - if !status.success() { - return Err(miette::miette!( - "dynamic compose child cleanup on site `{}` failed with status {status}", - plan.site_id - )); - } - Ok(()) -} - -fn rewrite_dynamic_proxy_metadata( - artifact_root: &Path, - site_plan: &DynamicSitePlanRecord, -) -> Result<()> { - if site_plan.proxy_exports.is_empty() { - return Ok(()); - } - if site_plan.kind == SiteKind::Compose { - let path = artifact_root.join("compose.yaml"); - let raw = fs::read_to_string(&path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid compose file {}", path.display()))?; - let Some(root) = document.as_mapping_mut() else { - return Err(miette::miette!( - "compose file {} is not a YAML mapping", - path.display() - )); - }; - let x_amber_key = serde_yaml::Value::String("x-amber".to_string()); - let Some(x_amber) = root.get_mut(&x_amber_key) else { - return Err(miette::miette!( - "compose file {} is missing x-amber metadata", - path.display() - )); - }; - let exports = site_plan - .proxy_exports - .iter() - .map(|(name, export)| { - Ok(( - serde_yaml::Value::String(name.clone()), - serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([ - ( - serde_yaml::Value::String("component".to_string()), - serde_yaml::Value::String(export.component.clone()), - ), - ( - serde_yaml::Value::String("provide".to_string()), - serde_yaml::Value::String(export.provide.clone()), - ), - ( - serde_yaml::Value::String("protocol".to_string()), - serde_yaml::Value::String(export.protocol.clone()), - ), - ( - serde_yaml::Value::String("router_mesh_port".to_string()), - serde_yaml::Value::Number(0u64.into()), - ), - ( - serde_yaml::Value::String("route_id".to_string()), - serde_yaml::Value::String(dynamic_proxy_export_route_id(name, export)?), - ), - ])), - )) - }) - .collect::>()?; - let mut metadata = x_amber.as_mapping().cloned().ok_or_else(|| { - miette::miette!( - "compose file {} has non-mapping x-amber metadata", - path.display() - ) - })?; - metadata.insert( - serde_yaml::Value::String("exports".to_string()), - serde_yaml::Value::Mapping(exports), - ); - *x_amber = serde_yaml::Value::Mapping(metadata); - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; - fs::write(&path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) - } else { - let exports = site_plan - .proxy_exports - .iter() - .map(|(name, export)| { - Ok(( - name.clone(), - JsonValue::Object(JsonMap::from_iter([ - ( - "component".to_string(), - JsonValue::String(export.component.clone()), - ), - ( - "provide".to_string(), - JsonValue::String(export.provide.clone()), - ), - ( - "protocol".to_string(), - JsonValue::String(export.protocol.clone()), - ), - ("router_mesh_port".to_string(), JsonValue::from(0u64)), - ( - "route_id".to_string(), - JsonValue::String(dynamic_proxy_export_route_id(name, export)?), - ), - ])), - )) - }) - .collect::>>()?; - let path = artifact_root.join("amber-proxy.json"); - let mut metadata: JsonValue = read_json(&path, "proxy metadata")?; - let Some(object) = metadata.as_object_mut() else { - return Err(miette::miette!( - "proxy metadata {} is not a JSON object", - path.display() - )); - }; - object.insert("exports".to_string(), JsonValue::Object(exports)); - write_json(&path, &metadata) - } -} - -fn rewrite_dynamic_compose_proxy_metadata( - artifact_root: &Path, - compose_project: &str, -) -> Result<()> { - let path = artifact_root.join("compose.yaml"); - let raw = fs::read_to_string(&path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid compose file {}", path.display()))?; - let Some(root) = document.as_mapping_mut() else { - return Err(miette::miette!( - "compose file {} is not a YAML mapping", - path.display() - )); - }; - let x_amber_key = serde_yaml::Value::String("x-amber".to_string()); - let Some(x_amber) = root.get_mut(&x_amber_key) else { - return Err(miette::miette!( - "compose file {} is missing x-amber metadata", - path.display() - )); - }; - let mut metadata: ProxyMetadata = serde_yaml::from_value(x_amber.clone()) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "compose file {} has invalid x-amber metadata", - path.display() - ) - })?; - if let Some(router) = metadata.router.as_mut() { - router.compose_project = Some(compose_project.to_string()); - } - *x_amber = serde_yaml::to_value(&metadata) - .into_diagnostic() - .wrap_err("failed to serialize compose proxy metadata")?; - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; - fs::write(&path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) -} - -fn yaml_string(value: &str) -> serde_yaml::Value { - serde_yaml::Value::String(value.to_string()) -} - -fn compose_services<'a>( - document: &'a serde_yaml::Value, - path: &Path, -) -> Result<&'a serde_yaml::Mapping> { - document - .as_mapping() - .and_then(|root| root.get(yaml_string("services"))) - .and_then(serde_yaml::Value::as_mapping) - .ok_or_else(|| { - miette::miette!( - "compose file {} is missing a services mapping", - path.display() - ) - }) -} - -fn compose_services_mut<'a>( - document: &'a mut serde_yaml::Value, - path: &Path, -) -> Result<&'a mut serde_yaml::Mapping> { - document - .as_mapping_mut() - .and_then(|root| root.get_mut(yaml_string("services"))) - .and_then(serde_yaml::Value::as_mapping_mut) - .ok_or_else(|| { - miette::miette!( - "compose file {} is missing a services mapping", - path.display() - ) - }) -} - -fn compose_networks_mut<'a>( - document: &'a mut serde_yaml::Value, - path: &Path, -) -> Result> { - let Some(root) = document.as_mapping_mut() else { - return Err(miette::miette!( - "compose file {} is not a mapping", - path.display() - )); - }; - Ok(root - .get_mut(yaml_string("networks")) - .and_then(serde_yaml::Value::as_mapping_mut)) -} - -fn assign_compose_egress_network_subnets( - artifact_dir: &Path, - run_id: &str, - site_id: &str, -) -> Result<()> { - let compose_path = artifact_dir.join("compose.yaml"); - if !compose_path.is_file() { - return Ok(()); - } - - let mut document = read_compose_document(&compose_path)?; - let Some(networks) = compose_networks_mut(&mut document, &compose_path)? else { - return Ok(()); - }; - - let mut used_subnets = networks - .values() - .filter_map(compose_network_subnet) - .map(str::to_string) - .collect::>(); - let mut changed = false; - - for (name, network) in networks.iter_mut() { - let Some(name) = name.as_str() else { - continue; - }; - if !name.starts_with("amber_egress_") || compose_network_subnet(network).is_some() { - continue; - } - let subnet = next_compose_egress_subnet(run_id, site_id, name, &used_subnets); - set_compose_network_subnet(network, &subnet)?; - used_subnets.insert(subnet); - changed = true; - } - - if !changed { - return Ok(()); - } - - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", compose_path.display()))?; - fs::write(&compose_path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", compose_path.display())) -} - -fn compose_network_subnet(network: &serde_yaml::Value) -> Option<&str> { - network - .as_mapping() - .and_then(|mapping| mapping.get(yaml_string("ipam"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|ipam| ipam.get(yaml_string("config"))) - .and_then(serde_yaml::Value::as_sequence) - .and_then(|configs| configs.first()) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|config| config.get(yaml_string("subnet"))) - .and_then(serde_yaml::Value::as_str) -} - -fn set_compose_network_subnet(network: &mut serde_yaml::Value, subnet: &str) -> Result<()> { - let Some(mapping) = network.as_mapping_mut() else { - return Err(miette::miette!( - "compose network definition is not a mapping" - )); - }; - mapping.insert( - yaml_string("ipam"), - serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( - yaml_string("config"), - serde_yaml::Value::Sequence(vec![serde_yaml::Value::Mapping( - serde_yaml::Mapping::from_iter([(yaml_string("subnet"), yaml_string(subnet))]), - )]), - )])), - ); - Ok(()) -} - -fn next_compose_egress_subnet( - run_id: &str, - site_id: &str, - network_name: &str, - used_subnets: &BTreeSet, -) -> String { - let digest = Sha256::digest(format!("{run_id}:{site_id}:{network_name}")); - let mut bytes = [0u8; 4]; - bytes.copy_from_slice(&digest[..4]); - let base = u32::from_be_bytes(bytes) % COMPOSE_EGRESS_SUBNET_COUNT; - for offset in 0..COMPOSE_EGRESS_SUBNET_COUNT { - let candidate = - compose_egress_subnet_from_index((base + offset) % COMPOSE_EGRESS_SUBNET_COUNT); - if !used_subnets.contains(&candidate) { - return candidate; - } - } - unreachable!("compose egress subnet pool exhausted") -} - -const COMPOSE_EGRESS_SUBNET_COUNT: u32 = 1 << 18; - -fn compose_egress_subnet_from_index(index: u32) -> String { - let second_octet = 64 + ((index >> 12) & 0x3f); - let third_octet = (index >> 4) & 0xff; - let fourth_octet = (index & 0x0f) << 4; - format!("100.{second_octet}.{third_octet}.{fourth_octet}/28") -} - -fn compose_service_names(document: &serde_yaml::Value, path: &Path) -> Result> { - Ok(compose_services(document, path)? - .keys() - .filter_map(serde_yaml::Value::as_str) - .map(str::to_string) - .collect()) -} - -fn compose_service_dependency_names(service: &serde_yaml::Value) -> Result> { - let Some(mapping) = service.as_mapping() else { - return Err(miette::miette!( - "compose service definition is not a mapping" - )); - }; - let Some(depends_on) = mapping.get(yaml_string("depends_on")) else { - return Ok(Vec::new()); - }; - if let Some(sequence) = depends_on.as_sequence() { - return Ok(sequence - .iter() - .filter_map(serde_yaml::Value::as_str) - .map(str::to_string) - .collect()); - } - if let Some(depends_on_map) = depends_on.as_mapping() { - return Ok(depends_on_map - .keys() - .filter_map(serde_yaml::Value::as_str) - .map(str::to_string) - .collect()); - } - Err(miette::miette!( - "compose service has invalid depends_on declaration" - )) -} - -fn retain_compose_service_dependencies( - service: &mut serde_yaml::Value, - keep_services: &BTreeSet, -) -> Result<()> { - let Some(mapping) = service.as_mapping_mut() else { - return Err(miette::miette!( - "compose service definition is not a mapping" - )); - }; - let depends_on_key = yaml_string("depends_on"); - let Some(depends_on) = mapping.get_mut(&depends_on_key) else { - return Ok(()); - }; - match depends_on { - serde_yaml::Value::Sequence(sequence) => { - sequence.retain(|value| { - value - .as_str() - .is_some_and(|name| keep_services.contains(name)) - }); - if sequence.is_empty() { - mapping.remove(&depends_on_key); - } - } - serde_yaml::Value::Mapping(depends_on_map) => { - depends_on_map.retain(|key, _| { - key.as_str() - .is_some_and(|name| keep_services.contains(name)) - }); - if depends_on_map.is_empty() { - mapping.remove(&depends_on_key); - } - } - _ => { - return Err(miette::miette!( - "compose service has invalid depends_on declaration" - )); - } - } - Ok(()) -} - -fn compose_dynamic_root_service_names( - artifact_root: &Path, - assigned_components: &[String], -) -> Result> { - let compose_path = artifact_root.join("compose.yaml"); - let document = read_compose_document(&compose_path)?; - let service_names = compose_service_names(&document, &compose_path)?; - let plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; - let assigned = assigned_components - .iter() - .map(String::as_str) - .collect::>(); - let mut out = BTreeSet::new(); - for target in plan.targets { - if !matches!(target.kind, MeshProvisionTargetKind::Component) - || !assigned.contains(target.config.identity.id.as_str()) - { - continue; - } - let MeshProvisionOutput::Filesystem { dir } = target.output else { - return Err(miette::miette!( - "compose component {} does not use filesystem mesh output", - target.config.identity.id - )); - }; - let sidecar = Path::new(&dir) - .file_name() - .and_then(|value| value.to_str()) - .ok_or_else(|| { - miette::miette!( - "compose component {} has invalid mesh output dir {}", - target.config.identity.id, - dir - ) - })? - .to_string(); - if service_names.contains(&sidecar) { - out.insert(sidecar.clone()); - } - if let Some(program) = sidecar.strip_suffix("-net") - && service_names.contains(program) - { - out.insert(program.to_string()); - } - } - if out.is_empty() { - return Err(miette::miette!( - "compose child artifact {} does not contain services for {:?}", - compose_path.display(), - assigned_components - )); - } - Ok(out) -} - -fn compose_live_service_names( - plan: &SiteControllerRuntimePlan, - published_children: &[SiteControllerRuntimeChildRecord], -) -> Result> { - let site_compose_path = Path::new(&plan.artifact_dir).join("compose.yaml"); - let site_document = read_compose_document(&site_compose_path)?; - let mut live_services = compose_service_names(&site_document, &site_compose_path)?; - for child in published_children { - let child_compose_path = Path::new(&child.artifact_root).join("compose.yaml"); - let child_document = read_compose_document(&child_compose_path)?; - live_services.extend(compose_service_names(&child_document, &child_compose_path)?); - } - Ok(live_services) -} - -fn compose_service_closure( - document: &serde_yaml::Value, - compose_path: &Path, - roots: &BTreeSet, -) -> Result> { - let services = compose_services(document, compose_path)?; - let mut closure = BTreeSet::new(); - let mut queue = roots.iter().cloned().collect::>(); - while let Some(service_name) = queue.pop() { - if !closure.insert(service_name.clone()) { - continue; - } - let service = services.get(yaml_string(&service_name)).ok_or_else(|| { - miette::miette!( - "compose file {} is missing service {}", - compose_path.display(), - service_name - ) - })?; - for dependency in compose_service_dependency_names(service)? { - queue.push(dependency); - } - } - Ok(closure) -} - -fn dynamic_compose_mesh_dir(service_name: &str) -> String { - format!("{DYNAMIC_COMPOSE_MESH_ROOT}/{service_name}") -} - -fn filter_dynamic_router_target( - router_target: &mut amber_mesh::MeshProvisionTarget, - kept_component_ids: &BTreeSet, -) { - router_target - .config - .peers - .retain(|peer| kept_component_ids.contains(&peer.id)); - router_target - .config - .inbound - .retain(|route| match &route.target { - InboundTarget::MeshForward { peer_id, .. } => kept_component_ids.contains(peer_id), - _ => route - .allowed_issuers - .iter() - .any(|issuer| kept_component_ids.contains(issuer)), - }); -} - -fn dynamic_proxy_export_mesh_protocol(export: &DynamicProxyExportRecord) -> Result { - let protocol = export - .protocol - .parse::() - .map_err(|err| miette::miette!("invalid dynamic proxy export protocol: {err}"))?; - mesh_protocol(protocol) -} - -fn dynamic_proxy_export_route_id( - export_name: &str, - export: &DynamicProxyExportRecord, -) -> Result { - Ok(router_dynamic_export_route_id( - &export.component, - export_name, - dynamic_proxy_export_mesh_protocol(export)?, - )) -} - -fn dynamic_input_route_mesh_protocol(input: &DynamicInputRouteRecord) -> Result { - let protocol = input - .protocol - .parse::() - .map_err(|err| miette::miette!("invalid dynamic routed-input protocol: {err}"))?; - mesh_protocol(protocol) -} - -fn dynamic_input_route_route_id(input: &DynamicInputRouteRecord, protocol: MeshProtocol) -> String { - match &input.target { - DynamicInputRouteTarget::ComponentProvide { provide } => { - component_route_id(&input.provider_component, provide, protocol) - } - } -} - -fn dynamic_input_route_capability(input: &DynamicInputRouteRecord) -> String { - match &input.target { - DynamicInputRouteTarget::ComponentProvide { provide } => provide.clone(), - } -} - -fn is_compose_component_sidecar_service(service_name: &str) -> bool { - service_name.ends_with("-net") -} - -fn overlay_peer_addr_map_from_ports(ports: &BTreeMap) -> BTreeMap { - ports - .iter() - .map(|(component, port)| (component.clone(), format!("127.0.0.1:{port}"))) - .collect() -} - -fn overlay_issuer_sets( - routed_inputs: &[DynamicInputRouteRecord], -) -> Result>> { - dynamic_route_issuer_grants(&[SiteControllerRuntimeChildRecord { - child_id: 0, - artifact_root: String::new(), - assigned_components: Vec::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: routed_inputs.to_vec(), - process_pid: None, - published: false, - }]) -} - -fn overlay_upsert_peer( - peers: &mut Vec, - peer_identities: &BTreeMap, - peer_id: &str, -) -> Result<()> { - if peers.iter().any(|peer| peer.id == peer_id) { - return Ok(()); - } - let identity = peer_identities.get(peer_id).ok_or_else(|| { - miette::miette!("dynamic route overlay is missing mesh identity for peer {peer_id}") - })?; - peers.push(MeshPeer { - id: identity.id.clone(), - public_key: identity.public_key, - }); - Ok(()) -} - -fn overlay_upsert_route(routes: &mut Vec, route: InboundRoute) { - if let Some(existing) = routes - .iter_mut() - .find(|existing| existing.route_id == route.route_id) - { - *existing = route; - } else { - routes.push(route); - } -} - -fn routed_input_overlay_route( - input: &DynamicInputRouteRecord, - provider_peer_addr: &str, - allowed_issuers: Vec, -) -> Result { - let protocol = dynamic_input_route_mesh_protocol(input)?; - let (target_route_id, capability) = match &input.target { - DynamicInputRouteTarget::ComponentProvide { provide } => ( - component_route_id(&input.provider_component, provide, protocol), - provide.clone(), - ), - }; - Ok(InboundRoute { - route_id: dynamic_input_route_route_id(input, protocol), - capability: dynamic_input_route_capability(input), - capability_kind: Some(input.capability_kind.clone()), - capability_profile: input.capability_profile.clone(), - protocol, - http_plugins: http_route_plugins_for_capability_kind( - Some(input.capability_kind.as_str()), - protocol, - ), - target: InboundTarget::MeshForward { - peer_addr: provider_peer_addr.to_string(), - peer_id: input.provider_component.clone(), - route_id: target_route_id, - capability, - }, - allowed_issuers, - }) -} - -fn augment_route_overlay_payload( - payload: &mut StoredRouteOverlayPayload, - proxy_exports: &BTreeMap, - routed_inputs: &[DynamicInputRouteRecord], - provider_peer_addrs: &BTreeMap, - peer_identities: &BTreeMap, - allowed_issuers_by_route: Option<&BTreeMap>>, - skip_missing_providers: bool, -) -> Result<()> { - for export in proxy_exports.values() { - overlay_upsert_peer(&mut payload.peers, peer_identities, &export.component)?; - } - add_dynamic_proxy_export_overlay_routes( - &mut payload.inbound_routes, - proxy_exports, - |export| { - provider_peer_addrs - .get(&export.component) - .cloned() - .ok_or_else(|| { - miette::miette!( - "dynamic route overlay is missing a live peer address for {}", - export.component - ) - }) - }, - )?; - - for input in routed_inputs { - let Some(provider_peer_addr) = provider_peer_addrs.get(&input.provider_component) else { - if skip_missing_providers { - continue; - } - return Err(miette::miette!( - "dynamic route overlay is missing a live peer address for {}", - input.provider_component - )); - }; - overlay_upsert_peer( - &mut payload.peers, - peer_identities, - &input.provider_component, - )?; - let route_id = - dynamic_input_route_route_id(input, dynamic_input_route_mesh_protocol(input)?); - let allowed_issuers = allowed_issuers_by_route - .and_then(|issuers| issuers.get(&route_id)) - .map(|issuers| issuers.iter().cloned().collect()) - .unwrap_or_default(); - overlay_upsert_route( - &mut payload.inbound_routes, - routed_input_overlay_route(input, provider_peer_addr, allowed_issuers)?, - ); - } - - Ok(()) -} - -fn routed_input_router_peer_addr(kind: SiteKind, router_mesh_port: Option) -> Result { - let router_mesh_port = router_mesh_port.ok_or_else(|| { - miette::miette!("site {kind:?} is missing its router mesh port for routed child inputs") - })?; - Ok(match kind { - SiteKind::Direct | SiteKind::Vm => format!("127.0.0.1:{router_mesh_port}"), - SiteKind::Compose => format!("{COMPOSE_ROUTER_SERVICE_NAME}:{router_mesh_port}"), - SiteKind::Kubernetes => { - format!("{KUBERNETES_ROUTER_COMPONENT_NAME}:{router_mesh_port}") - } - }) -} - -fn router_mesh_port_from_plan(mesh_plan: &MeshProvisionPlan, artifact_kind: &str) -> Result { - mesh_plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - .map(|target| target.config.mesh_listen.port()) - .ok_or_else(|| { - miette::miette!("{artifact_kind} mesh provision plan is missing its router mesh target") - }) -} - -fn rewrite_dynamic_routed_inputs( - mesh_plan: &mut MeshProvisionPlan, - routed_inputs: &[DynamicInputRouteRecord], - kind: SiteKind, - router_identity_id: &str, - router_mesh_port: Option, -) -> Result<()> { - if routed_inputs.is_empty() { - return Ok(()); - } - - let router_peer_addr = routed_input_router_peer_addr(kind, router_mesh_port)?; - for input in routed_inputs { - let protocol = dynamic_input_route_mesh_protocol(input)?; - let component_target = mesh_plan - .targets - .iter_mut() - .find(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && target.config.identity.id == input.component - }) - .ok_or_else(|| { - miette::miette!( - "dynamic routed input {}.{} is missing component {} in the mesh provision plan", - input.component, - input.slot, - input.component - ) - })?; - if !component_target - .config - .peers - .iter() - .any(|peer| peer.id == router_identity_id) - { - component_target - .config - .peers - .push(amber_mesh::MeshPeerTemplate { - id: router_identity_id.to_string(), - }); - } - - let route_id = dynamic_input_route_route_id(input, protocol); - let capability = dynamic_input_route_capability(input); - let mut matched = false; - for route in component_target - .config - .outbound - .iter_mut() - .filter(|route| route.slot == input.slot) - { - matched = true; - route.route_id = route_id.clone(); - route.protocol = protocol; - route.peer_addr = router_peer_addr.clone(); - route.peer_id = router_identity_id.to_string(); - route.capability = capability.clone(); - route.capability_kind = Some(input.capability_kind.clone()); - route.capability_profile = input.capability_profile.clone(); - route.http_plugins = http_route_plugins_for_capability_kind( - Some(input.capability_kind.as_str()), - protocol, - ); - } - if !matched { - return Err(miette::miette!( - "dynamic routed input {}.{} is missing an outbound route in the mesh provision \ - plan", - input.component, - input.slot - )); - } - } - - Ok(()) -} - -fn rewrite_dynamic_routed_inputs_in_artifact( - artifact_root: &Path, - site_plan: &DynamicSitePlanRecord, - router_mesh_port: Option, -) -> Result<()> { - if site_plan.routed_inputs.is_empty() { - return Ok(()); - } - let path = artifact_root.join("mesh-provision-plan.json"); - let mut mesh_plan: MeshProvisionPlan = read_json(&path, "mesh provision plan")?; - rewrite_dynamic_routed_inputs( - &mut mesh_plan, - &site_plan.routed_inputs, - site_plan.kind, - &site_plan.router_identity_id, - router_mesh_port, - )?; - write_json(&path, &mesh_plan) -} - -fn build_filesystem_route_overlay_base( - artifact_root: &Path, - assigned_components: &[String], - provider_peer_addrs: &BTreeMap, - peer_identities: &BTreeMap, -) -> Result { - let mesh_plan: MeshProvisionPlan = read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - )?; - let kept_component_ids = assigned_components.iter().cloned().collect::>(); - let mut router_target = mesh_plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - .cloned() - .ok_or_else(|| { - miette::miette!( - "dynamic artifact {} is missing a router mesh target", - artifact_root.display() - ) - })?; - filter_dynamic_router_target(&mut router_target, &kept_component_ids); - for route in &mut router_target.config.inbound { - if let InboundTarget::MeshForward { - peer_id, peer_addr, .. - } = &mut route.target - && let Some(resolved) = provider_peer_addrs.get(peer_id) - { - *peer_addr = resolved.clone(); - } - } - let peers = router_target - .config - .peers - .iter() - .map(|peer| { - peer_identities.get(&peer.id).map(|identity| MeshPeer { - id: identity.id.clone(), - public_key: identity.public_key, - }) - }) - .collect::>>() - .ok_or_else(|| { - miette::miette!( - "dynamic artifact {} is missing a live mesh identity for one of its route peers", - artifact_root.display() - ) - })?; - Ok(StoredRouteOverlayPayload { - peers, - inbound_routes: router_target.config.inbound, - }) -} - -fn write_direct_vm_startup_route_overlay_payload( - artifact_root: &Path, - _artifact_kind: &str, - routed_inputs: &[DynamicInputRouteRecord], - provider_peer_addrs: &BTreeMap, - existing_site_peer_identities: &BTreeMap, -) -> Result<()> { - if routed_inputs.is_empty() { - return Ok(()); - } - let allowed_issuers = overlay_issuer_sets(routed_inputs)?; - let mut payload = StoredRouteOverlayPayload { - peers: Vec::new(), - inbound_routes: Vec::new(), - }; - augment_route_overlay_payload( - &mut payload, - &BTreeMap::new(), - routed_inputs, - provider_peer_addrs, - existing_site_peer_identities, - Some(&allowed_issuers), - true, - )?; - if payload.inbound_routes.is_empty() { - return Ok(()); - } - write_dynamic_route_overlay_payload(artifact_root, &payload) -} - -fn write_direct_vm_live_route_overlay_payload( - artifact_root: &Path, - assigned_components: &[String], - proxy_exports: &BTreeMap, - routed_inputs: &[DynamicInputRouteRecord], - provider_peer_addrs: &BTreeMap, - peer_identities: &BTreeMap, -) -> Result<()> { - let mut payload = build_filesystem_route_overlay_base( - artifact_root, - assigned_components, - provider_peer_addrs, - peer_identities, - )?; - augment_route_overlay_payload( - &mut payload, - proxy_exports, - routed_inputs, - provider_peer_addrs, - peer_identities, - None, - false, - )?; - write_dynamic_route_overlay_payload(artifact_root, &payload) -} - -fn ensure_dynamic_proxy_export_component_routes( - mesh_plan: &mut MeshProvisionPlan, - proxy_exports: &BTreeMap, - router_identity_id: &str, -) -> Result<()> { - for export in proxy_exports.values() { - let protocol = dynamic_proxy_export_mesh_protocol(export)?; - let route_id = component_route_id(&export.component, &export.provide, protocol); - let component_target = mesh_plan - .targets - .iter_mut() - .find(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && target.config.identity.id == export.component - }) - .ok_or_else(|| { - miette::miette!( - "dynamic proxy export provider {} is missing from the mesh provision plan", - export.component - ) - })?; - if !component_target - .config - .peers - .iter() - .any(|peer| peer.id == router_identity_id) - { - component_target - .config - .peers - .push(amber_mesh::MeshPeerTemplate { - id: router_identity_id.to_string(), - }); - } - if let Some(route) = component_target - .config - .inbound - .iter_mut() - .find(|route| route.route_id == route_id) - { - if !route - .allowed_issuers - .iter() - .any(|issuer| issuer == router_identity_id) - { - route.allowed_issuers.push(router_identity_id.to_string()); - route.allowed_issuers.sort(); - route.allowed_issuers.dedup(); - } - continue; - } - component_target.config.inbound.push(InboundRoute { - route_id, - capability: export.provide.clone(), - capability_kind: Some(export.capability_kind.clone()), - capability_profile: export.capability_profile.clone(), - protocol, - http_plugins: http_route_plugins_for_capability_kind( - Some(export.capability_kind.as_str()), - protocol, - ), - target: InboundTarget::Local { - port: export.target_port, - }, - allowed_issuers: vec![router_identity_id.to_string()], - }); - } - Ok(()) -} - -fn ensure_dynamic_proxy_export_component_routes_in_artifact( - artifact_root: &Path, - proxy_exports: &BTreeMap, - router_identity_id: &str, -) -> Result<()> { - let plan_path = artifact_root.join("mesh-provision-plan.json"); - let mut mesh_plan: MeshProvisionPlan = read_json(&plan_path, "mesh provision plan")?; - ensure_dynamic_proxy_export_component_routes( - &mut mesh_plan, - proxy_exports, - router_identity_id, - )?; - write_json(&plan_path, &mesh_plan) -} - -fn add_dynamic_proxy_export_overlay_routes( - inbound_routes: &mut Vec, - proxy_exports: &BTreeMap, - mut peer_addr_for_export: impl FnMut(&DynamicProxyExportRecord) -> Result, -) -> Result<()> { - for (export_name, export) in proxy_exports { - let protocol = dynamic_proxy_export_mesh_protocol(export)?; - let route_id = router_dynamic_export_route_id(&export.component, export_name, protocol); - let route = InboundRoute { - route_id, - capability: export_name.clone(), - capability_kind: Some(export.capability_kind.clone()), - capability_profile: export.capability_profile.clone(), - protocol, - http_plugins: http_route_plugins_for_capability_kind( - Some(export.capability_kind.as_str()), - protocol, - ), - target: InboundTarget::MeshForward { - peer_addr: peer_addr_for_export(export)?, - peer_id: export.component.clone(), - route_id: component_route_id(&export.component, &export.provide, protocol), - capability: export.provide.clone(), - }, - allowed_issuers: Vec::new(), - }; - if let Some(existing) = inbound_routes - .iter_mut() - .find(|existing| existing.route_id == route.route_id) - { - *existing = route; - } else { - inbound_routes.push(route); - } - } - Ok(()) -} - -struct DynamicComposeMeshPlan { - mesh_plan: MeshProvisionPlan, - mesh_dirs: BTreeMap, - component_mesh_dirs: BTreeMap, -} - -fn build_dynamic_compose_mesh_plan( - artifact_root: &Path, - assigned_components: &[String], -) -> Result { - let plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; - let assigned = assigned_components - .iter() - .map(String::as_str) - .collect::>(); - let mut filtered_targets = Vec::new(); - let mut mesh_dirs = BTreeMap::new(); - let mut component_mesh_dirs = BTreeMap::new(); - - for mut target in plan.targets { - match target.kind { - MeshProvisionTargetKind::Component => { - if !assigned.contains(target.config.identity.id.as_str()) { - continue; - } - let MeshProvisionOutput::Filesystem { dir } = &mut target.output else { - return Err(miette::miette!( - "compose component {} does not use filesystem mesh output", - target.config.identity.id - )); - }; - let sidecar = Path::new(dir) - .file_name() - .and_then(|value| value.to_str()) - .ok_or_else(|| { - miette::miette!( - "compose component {} has invalid mesh output dir {}", - target.config.identity.id, - dir - ) - })? - .to_string(); - let relative_dir = dynamic_compose_mesh_dir(&sidecar); - *dir = relative_dir.clone(); - component_mesh_dirs.insert(target.config.identity.id.clone(), relative_dir.clone()); - mesh_dirs.insert(sidecar, relative_dir); - filtered_targets.push(target); - } - MeshProvisionTargetKind::Router => {} - } - } - - if filtered_targets.is_empty() { - return Err(miette::miette!( - "compose child artifact {} does not contain assigned child mesh targets", - artifact_root.join("compose.yaml").display() - )); - } - - Ok(DynamicComposeMeshPlan { - mesh_plan: MeshProvisionPlan { - version: plan.version, - identity_seed: plan.identity_seed, - existing_peer_identities: Vec::new(), - targets: filtered_targets, - }, - mesh_dirs, - component_mesh_dirs, - }) -} - -fn compose_component_mesh_peer_addr( - artifact_root: &Path, - component_id: &str, - output: &MeshProvisionOutput, - mesh_port: u16, -) -> Result { - let MeshProvisionOutput::Filesystem { dir } = output else { - return Err(miette::miette!( - "compose artifact {} component {} does not use filesystem mesh output", - artifact_root.display(), - component_id - )); - }; - let service_name = Path::new(dir) - .file_name() - .and_then(|value| value.to_str()) - .ok_or_else(|| { - miette::miette!( - "compose artifact {} component {} has invalid mesh output dir {}", - artifact_root.display(), - component_id, - dir - ) - })?; - Ok(format!("{service_name}:{mesh_port}")) -} - -fn kubernetes_component_mesh_peer_addr( - artifact_root: &Path, - component_id: &str, - output: &MeshProvisionOutput, - mesh_port: u16, -) -> Result { - let MeshProvisionOutput::KubernetesSecret { name, .. } = output else { - return Err(miette::miette!( - "kubernetes artifact {} component {} does not use a kubernetes secret mesh output", - artifact_root.display(), - component_id - )); - }; - let service_name = name.strip_suffix("-mesh").ok_or_else(|| { - miette::miette!( - "kubernetes artifact {} component {} uses invalid mesh secret name {}", - artifact_root.display(), - component_id, - name - ) - })?; - Ok(format!("{service_name}:{mesh_port}")) -} - -fn build_dynamic_compose_route_overlay_payload( - artifact_root: &Path, - assigned_components: &[String], - component_mesh_dirs: &BTreeMap, - proxy_exports: &BTreeMap, - routed_inputs: &[DynamicInputRouteRecord], - existing_site_peer_identities: &BTreeMap, -) -> Result { - let plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; - let assigned = assigned_components - .iter() - .map(String::as_str) - .collect::>(); - let kept_component_ids = plan - .targets - .iter() - .filter(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && assigned.contains(target.config.identity.id.as_str()) - }) - .map(|target| target.config.identity.id.clone()) - .collect::>(); - let mut router_target = plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - .cloned() - .ok_or_else(|| { - miette::miette!( - "compose child artifact {} is missing a router mesh target", - artifact_root.join("compose.yaml").display() - ) - })?; - filter_dynamic_router_target(&mut router_target, &kept_component_ids); - let component_peer_addrs = plan - .targets - .iter() - .filter(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - .map(|target| { - Ok(( - target.config.identity.id.clone(), - compose_component_mesh_peer_addr( - artifact_root, - &target.config.identity.id, - &target.output, - target.config.mesh_listen.port(), - )?, - )) - }) - .collect::>>()?; - for route in &mut router_target.config.inbound { - if let InboundTarget::MeshForward { - peer_id, peer_addr, .. - } = &mut route.target - && let Some(resolved) = component_peer_addrs.get(peer_id) - { - *peer_addr = resolved.clone(); - } - } - - let component_mesh_scopes = plan - .targets - .iter() - .filter(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - .map(|target| { - ( - target.config.identity.id.clone(), - target.config.identity.mesh_scope.clone(), - ) - }) - .collect::>(); - let mut peer_identities = existing_site_peer_identities.clone(); - for (component, relative_dir) in component_mesh_dirs { - let identity: MeshIdentitySecret = read_json( - &artifact_root - .join(relative_dir) - .join(MESH_IDENTITY_FILENAME), - "mesh identity", - )?; - peer_identities.insert( - component.clone(), - MeshIdentityPublic { - id: identity.id.clone(), - public_key: identity.public_key().into_diagnostic()?, - mesh_scope: component_mesh_scopes.get(component).cloned().flatten(), - }, - ); - } - let peers = router_target - .config - .peers - .iter() - .map(|peer| { - let identity = peer_identities.get(&peer.id).ok_or_else(|| { - miette::miette!( - "compose child router overlay peer {} is missing a live mesh identity", - peer.id - ) - })?; - Ok(MeshPeer { - id: identity.id.clone(), - public_key: identity.public_key, - }) - }) - .collect::>>()?; - - let allowed_issuers = overlay_issuer_sets(routed_inputs)?; - let mut payload = StoredRouteOverlayPayload { - peers, - inbound_routes: router_target.config.inbound, - }; - augment_route_overlay_payload( - &mut payload, - proxy_exports, - routed_inputs, - &component_peer_addrs, - &peer_identities, - Some(&allowed_issuers), - false, - )?; - Ok(payload) -} - -fn rewrite_compose_mesh_bind_mounts( - artifact_root: &Path, - mesh_dirs: &BTreeMap, -) -> Result<()> { - let compose_path = artifact_root.join("compose.yaml"); - let mut document = read_compose_document(&compose_path)?; - let services = compose_services_mut(&mut document, &compose_path)?; - for (service_name, relative_dir) in mesh_dirs { - let service = services.get_mut(yaml_string(service_name)).ok_or_else(|| { - miette::miette!( - "compose file {} is missing child sidecar service {}", - compose_path.display(), - service_name - ) - })?; - let Some(service_mapping) = service.as_mapping_mut() else { - return Err(miette::miette!( - "compose service {} is not a mapping", - service_name - )); - }; - let volumes_key = yaml_string("volumes"); - let volumes = service_mapping - .get_mut(&volumes_key) - .and_then(serde_yaml::Value::as_sequence_mut) - .ok_or_else(|| { - miette::miette!( - "compose child sidecar {} is missing a volumes list", - service_name - ) - })?; - let expected_prefix = format!("{service_name}-mesh:/amber/mesh"); - let replacement = serde_yaml::Value::String(format!("./{relative_dir}:/amber/mesh:ro")); - let mut replaced = false; - for volume in volumes.iter_mut() { - if volume - .as_str() - .is_some_and(|value| value.starts_with(&expected_prefix)) - { - *volume = replacement.clone(); - replaced = true; - } - } - if !replaced { - volumes.push(replacement); - } - } - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", compose_path.display()))?; - fs::write(&compose_path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", compose_path.display())) -} - -fn load_dynamic_compose_child_metadata( - artifact_root: &Path, -) -> Result { - read_json( - &dynamic_compose_child_metadata_path(artifact_root), - "dynamic compose child metadata", - ) -} - -fn load_running_site_router_identity( - plan: &SiteControllerRuntimePlan, -) -> Result { - let state: SiteManagerState = read_json( - &Path::new(&plan.site_state_root).join("manager-state.json"), - "site manager state", - )?; - let router_identity_id = state.router_identity_id.ok_or_else(|| { - miette::miette!( - "site `{}` manager state is missing router identity id", - plan.site_id - ) - })?; - let router_public_key_b64 = state.router_public_key_b64.ok_or_else(|| { - miette::miette!( - "site `{}` manager state is missing router public key", - plan.site_id - ) - })?; - Ok(MeshIdentityPublic { - id: router_identity_id, - public_key: decode_public_key(&router_public_key_b64)?, - mesh_scope: Some(plan.mesh_scope.clone()), - }) -} - -fn filesystem_component_peer_identities_for_artifact( - artifact_root: &Path, - mesh_plan: &MeshProvisionPlan, - artifact_kind: &str, - compose_project: Option<&str>, -) -> Result> { - let mut peers = BTreeMap::new(); - for target in &mesh_plan.targets { - if !matches!(target.kind, MeshProvisionTargetKind::Component) { - continue; - } - let MeshProvisionOutput::Filesystem { dir } = &target.output else { - return Err(miette::miette!( - "{artifact_kind} artifact {} has non-filesystem mesh output for component {}", - artifact_root.display(), - target.config.identity.id - )); - }; - let config: MeshConfigPublic = if Path::new(dir).is_absolute() { - let compose_project = compose_project.ok_or_else(|| { - miette::miette!( - "{artifact_kind} artifact {} uses absolute mesh output {} without a compose \ - project", - artifact_root.display(), - dir - ) - })?; - let service_name = Path::new(dir) - .file_name() - .and_then(|value| value.to_str()) - .ok_or_else(|| { - miette::miette!( - "{artifact_kind} artifact {} has invalid absolute mesh output {}", - artifact_root.display(), - dir - ) - })?; - read_compose_volume_mesh_config(compose_project, service_name)? - } else { - read_json( - &artifact_root.join(dir).join(MESH_CONFIG_FILENAME), - "mesh config", - )? - }; - peers.insert(config.identity.id.clone(), config.identity); - } - Ok(peers) -} - -fn read_compose_volume_mesh_config( - compose_project: &str, - service_name: &str, -) -> Result { - let volume_name = format!("{compose_project}_{service_name}-mesh"); - let output = Command::new("docker") - .arg("run") - .arg("--rm") - .arg("-v") - .arg(format!("{volume_name}:/amber/mesh:ro")) - .arg("busybox:1.36.1") - .arg("cat") - .arg(format!("/amber/mesh/{MESH_CONFIG_FILENAME}")) - .output() - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to read compose mesh config from docker volume {volume_name}") - })?; - if !output.status.success() { - return Err(miette::miette!( - "failed to read compose mesh config from docker volume \ - {volume_name}\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - )); - } - serde_json::from_slice(&output.stdout) - .into_diagnostic() - .wrap_err_with(|| format!("docker volume {volume_name} returned invalid mesh config json")) -} - -fn compose_peer_identities_for_artifact( - artifact_root: &Path, - compose_project: Option<&str>, -) -> Result> { - let mesh_plan = if artifact_root.join("mesh-provision-plan.json").is_file() { - read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - )? - } else { - read_embedded_compose_mesh_provision_plan(artifact_root)? - }; - filesystem_component_peer_identities_for_artifact( - artifact_root, - &mesh_plan, - "compose", - compose_project, - ) -} - -fn local_compose_peer_identities( - plan: &SiteControllerRuntimePlan, - published_children: &[SiteControllerRuntimeChildRecord], -) -> Result> { - let mut peers = BTreeMap::new(); - let router = load_running_site_router_identity(plan)?; - peers.insert(router.id.clone(), router); - peers.extend(compose_peer_identities_for_artifact( - Path::new(&plan.artifact_dir), - plan.compose_project.as_deref(), - )?); - for child in published_children { - peers.extend(compose_peer_identities_for_artifact( - Path::new(&child.artifact_root), - plan.compose_project.as_deref(), - )?); - } - Ok(peers) -} - -fn kubernetes_peer_identities_for_artifact( - plan: &SiteControllerRuntimePlan, - artifact_root: &Path, -) -> Result> { - let mesh_plan = read_embedded_kubernetes_mesh_provision_plan(artifact_root)?; - let mut peers = BTreeMap::new(); - for target in &mesh_plan.targets { - if !matches!(target.kind, MeshProvisionTargetKind::Component) { - continue; - } - let MeshProvisionOutput::KubernetesSecret { name, namespace } = &target.output else { - return Err(miette::miette!( - "kubernetes artifact {} has non-secret mesh output for component {}", - artifact_root.display(), - target.config.identity.id - )); - }; - let config = load_kubernetes_mesh_config_public(plan, name, namespace.as_deref())?; - peers.insert(config.identity.id.clone(), config.identity); - } - Ok(peers) -} - -fn local_kubernetes_peer_identities( - plan: &SiteControllerRuntimePlan, - published_children: &[SiteControllerRuntimeChildRecord], -) -> Result> { - let mut peers = BTreeMap::new(); - let router = load_running_site_router_identity(plan)?; - peers.insert(router.id.clone(), router); - peers.extend(kubernetes_peer_identities_for_artifact( - plan, - Path::new(&plan.artifact_dir), - )?); - for child in published_children { - peers.extend(kubernetes_peer_identities_for_artifact( - plan, - Path::new(&child.artifact_root), - )?); - } - Ok(peers) -} - -pub(crate) fn collect_live_component_runtime_metadata( - plan: &SiteControllerRuntimePlan, -) -> Result> { - let state: SiteControllerRuntimeState = read_json( - &site_controller_runtime_state_path(Path::new(&plan.site_state_root)), - "site controller runtime state", - )?; - let published_children = state - .children - .values() - .filter(|child| child.published) - .cloned() - .collect::>(); - let mut components = match plan.kind { - SiteKind::Direct => collect_direct_artifact_runtime_metadata( - Path::new(&plan.artifact_dir), - Path::new(plan.runtime_root.as_deref().ok_or_else(|| { - miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) - })?), - )?, - SiteKind::Vm => collect_vm_artifact_runtime_metadata( - Path::new(&plan.artifact_dir), - Path::new(plan.runtime_root.as_deref().ok_or_else(|| { - miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) - })?), - )?, - SiteKind::Compose => collect_compose_artifact_runtime_metadata( - Path::new(&plan.artifact_dir), - plan.compose_project.as_deref(), - )?, - SiteKind::Kubernetes => { - collect_kubernetes_artifact_runtime_metadata(plan, Path::new(&plan.artifact_dir))? - } - }; - for child in &published_children { - let child_components = match plan.kind { - SiteKind::Direct => collect_direct_artifact_runtime_metadata( - Path::new(&child.artifact_root), - &site_controller_runtime_child_runtime_root(plan, child.child_id), - )?, - SiteKind::Vm => collect_vm_artifact_runtime_metadata( - Path::new(&child.artifact_root), - &site_controller_runtime_child_runtime_root(plan, child.child_id), - )?, - SiteKind::Compose => collect_compose_artifact_runtime_metadata( - Path::new(&child.artifact_root), - plan.compose_project.as_deref(), - )?, - SiteKind::Kubernetes => { - collect_kubernetes_artifact_runtime_metadata(plan, Path::new(&child.artifact_root))? - } - }; - components.extend(child_components); - } - Ok(components) -} - -pub(crate) fn load_live_site_router_mesh_config( - plan: &SiteControllerRuntimePlan, -) -> Result { - let artifact_root = Path::new(&plan.artifact_dir); - match plan.kind { - SiteKind::Direct => { - let runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { - miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) - })?); - let direct_plan: DirectPlan = - read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; - let router = direct_plan.router.ok_or_else(|| { - miette::miette!("direct site `{}` is missing its router plan", plan.site_id) - })?; - read_json(&runtime_root.join(&router.mesh_config_path), "mesh config") - } - SiteKind::Vm => { - let runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { - miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) - })?); - let vm_plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; - let router = vm_plan.router.ok_or_else(|| { - miette::miette!("vm site `{}` is missing its router plan", plan.site_id) - })?; - read_json(&runtime_root.join(&router.mesh_config_path), "mesh config") - } - SiteKind::Compose => { - let mesh_plan = if artifact_root.join("mesh-provision-plan.json").is_file() { - read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - )? - } else { - read_embedded_compose_mesh_provision_plan(artifact_root)? - }; - let target = mesh_plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - .ok_or_else(|| { - miette::miette!( - "compose site `{}` is missing a router mesh target", - plan.site_id - ) - })?; - let MeshProvisionOutput::Filesystem { dir } = &target.output else { - return Err(miette::miette!( - "compose site `{}` has non-filesystem mesh output for router {}", - plan.site_id, - target.config.identity.id - )); - }; - if Path::new(dir).is_absolute() { - let compose_project = plan.compose_project.as_deref().ok_or_else(|| { - miette::miette!( - "compose site `{}` is missing its compose project", - plan.site_id - ) - })?; - let service_name = Path::new(dir) - .file_name() - .and_then(|value| value.to_str()) - .ok_or_else(|| { - miette::miette!( - "compose site `{}` has invalid router mesh output {}", - plan.site_id, - dir - ) - })?; - read_compose_volume_mesh_config(compose_project, service_name) - } else { - read_json( - &artifact_root.join(dir).join(MESH_CONFIG_FILENAME), - "mesh config", - ) - } - } - SiteKind::Kubernetes => { - let mesh_plan = read_kubernetes_runtime_mesh_provision_plan(artifact_root)?; - let target = mesh_plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - .ok_or_else(|| { - miette::miette!( - "kubernetes site `{}` is missing a router mesh target", - plan.site_id - ) - })?; - let MeshProvisionOutput::KubernetesSecret { name, namespace } = &target.output else { - return Err(miette::miette!( - "kubernetes site `{}` has non-secret mesh output for router {}", - plan.site_id, - target.config.identity.id - )); - }; - load_kubernetes_mesh_config_public(plan, name, namespace.as_deref()) - } - } -} - -fn collect_direct_artifact_runtime_metadata( - artifact_root: &Path, - runtime_root: &Path, -) -> Result> { - let state: crate::direct_runtime::DirectRuntimeState = read_json( - &direct_runtime_state_path(artifact_root), - "direct runtime state", - )?; - let plan: DirectPlan = read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; - let mut components = BTreeMap::new(); - for component in &plan.components { - let mesh_config: MeshConfigPublic = read_json( - &runtime_root.join(&component.sidecar.mesh_config_path), - "mesh config", - )?; - let mesh_port = state - .component_mesh_port_by_id - .get(&component.id) - .copied() - .ok_or_else(|| { - miette::miette!( - "direct runtime state is missing mesh port for component {}", - component.moniker - ) - })?; - components.insert( - component.moniker.clone(), - LiveComponentRuntimeMetadata { - moniker: component.moniker.clone(), - host_mesh_addr: format!("127.0.0.1:{mesh_port}"), - mesh_config, - }, - ); - } - Ok(components) -} - -fn collect_vm_artifact_runtime_metadata( - artifact_root: &Path, - runtime_root: &Path, -) -> Result> { - let state = load_vm_runtime_state_for_artifact(artifact_root, runtime_root)?; - let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; - let mut components = BTreeMap::new(); - for component in &plan.components { - let mesh_config: MeshConfigPublic = read_json( - &runtime_root.join(&component.mesh_config_path), - "mesh config", - )?; - let mesh_port = state - .component_mesh_port_by_id - .get(&component.id) - .copied() - .ok_or_else(|| { - miette::miette!( - "vm runtime state is missing mesh port for component {}", - component.moniker - ) - })?; - components.insert( - component.moniker.clone(), - LiveComponentRuntimeMetadata { - moniker: component.moniker.clone(), - host_mesh_addr: format!("127.0.0.1:{mesh_port}"), - mesh_config, - }, - ); - } - Ok(components) -} - -fn collect_compose_artifact_runtime_metadata( - artifact_root: &Path, - compose_project: Option<&str>, -) -> Result> { - let mesh_plan = if artifact_root.join("mesh-provision-plan.json").is_file() { - read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - )? - } else { - read_embedded_compose_mesh_provision_plan(artifact_root)? - }; - let mut components = BTreeMap::new(); - for target in &mesh_plan.targets { - if !matches!(target.kind, MeshProvisionTargetKind::Component) { - continue; - } - let MeshProvisionOutput::Filesystem { dir } = &target.output else { - return Err(miette::miette!( - "compose artifact {} has non-filesystem mesh output for component {}", - artifact_root.display(), - target.config.identity.id - )); - }; - let service_name = Path::new(dir) - .file_name() - .and_then(|value| value.to_str()) - .ok_or_else(|| { - miette::miette!( - "compose artifact {} has invalid mesh output {} for component {}", - artifact_root.display(), - dir, - target.config.identity.id - ) - })?; - let mesh_config = if Path::new(dir).is_absolute() { - let compose_project = compose_project.ok_or_else(|| { - miette::miette!( - "compose artifact {} requires a compose project to resolve mesh output {}", - artifact_root.display(), - dir - ) - })?; - read_compose_volume_mesh_config(compose_project, service_name)? - } else { - read_json( - &artifact_root.join(dir).join(MESH_CONFIG_FILENAME), - "mesh config", - )? - }; - components.insert( - target.config.identity.id.clone(), - LiveComponentRuntimeMetadata { - moniker: target.config.identity.id.clone(), - host_mesh_addr: compose_component_mesh_peer_addr( - artifact_root, - &target.config.identity.id, - &target.output, - mesh_config.mesh_listen.port(), - )?, - mesh_config, - }, - ); - } - Ok(components) -} - -fn collect_kubernetes_artifact_runtime_metadata( - plan: &SiteControllerRuntimePlan, - artifact_root: &Path, -) -> Result> { - let mesh_plan = read_kubernetes_runtime_mesh_provision_plan(artifact_root)?; - let mut components = BTreeMap::new(); - for target in &mesh_plan.targets { - if !matches!(target.kind, MeshProvisionTargetKind::Component) { - continue; - } - let MeshProvisionOutput::KubernetesSecret { name, namespace } = &target.output else { - return Err(miette::miette!( - "kubernetes artifact {} has non-secret mesh output for component {}", - artifact_root.display(), - target.config.identity.id - )); - }; - name.strip_suffix("-mesh").ok_or_else(|| { - miette::miette!( - "kubernetes artifact {} component {} uses invalid mesh secret name {}", - artifact_root.display(), - target.config.identity.id, - name - ) - })?; - let mesh_config = load_kubernetes_mesh_config_public(plan, name, namespace.as_deref())?; - components.insert( - target.config.identity.id.clone(), - LiveComponentRuntimeMetadata { - moniker: target.config.identity.id.clone(), - host_mesh_addr: kubernetes_component_mesh_peer_addr( - artifact_root, - &target.config.identity.id, - &target.output, - mesh_config.mesh_listen.port(), - )?, - mesh_config, - }, - ); - } - Ok(components) -} - -fn prepare_dynamic_compose_child_artifact( - plan: &SiteControllerRuntimePlan, - site_plan: &DynamicSitePlanRecord, - artifact_root: &Path, - published_children: &[SiteControllerRuntimeChildRecord], - existing_site_peer_identities: &BTreeMap, -) -> Result<()> { - project_dynamic_child_mesh_scope(artifact_root, Some(&plan.mesh_scope))?; - let compose_path = artifact_root.join("compose.yaml"); - let mut desired_document = read_compose_document(&compose_path)?; - let root_services = - compose_dynamic_root_service_names(artifact_root, &site_plan.assigned_components)?; - let service_closure = - compose_service_closure(&desired_document, &compose_path, &root_services)?; - let live_services = compose_live_service_names(plan, published_children)?; - let mut kept_services = service_closure - .difference(&live_services) - .cloned() - .collect::>(); - kept_services.remove(COMPOSE_PROVISIONER_SERVICE_NAME); - if kept_services.is_empty() { - return Err(miette::miette!( - "compose child artifact {} does not retain any child-owned services after filtering", - compose_path.display() - )); - } - - let services = compose_services_mut(&mut desired_document, &compose_path)?; - services.retain(|name, _| { - name.as_str() - .is_some_and(|service_name| kept_services.contains(service_name)) - }); - for service in services.values_mut() { - retain_compose_service_dependencies(service, &kept_services)?; - } - let rendered = serde_yaml::to_string(&desired_document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", compose_path.display()))?; - fs::write(&compose_path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", compose_path.display()))?; - - let DynamicComposeMeshPlan { - mesh_plan, - mesh_dirs, - component_mesh_dirs, - } = build_dynamic_compose_mesh_plan(artifact_root, &site_plan.assigned_components)?; - let mut mesh_plan = mesh_plan; - let router_mesh_port = router_mesh_port_from_plan( - &read_embedded_compose_mesh_provision_plan(artifact_root)?, - "compose", - )?; - ensure_dynamic_proxy_export_component_routes( - &mut mesh_plan, - &site_plan.proxy_exports, - &plan.router_identity_id, - )?; - rewrite_dynamic_routed_inputs( - &mut mesh_plan, - &site_plan.routed_inputs, - SiteKind::Compose, - &plan.router_identity_id, - Some(router_mesh_port), - )?; - let existing_peer_identities = crate::direct_runtime::required_existing_mesh_peer_identities( - &mesh_plan, - existing_site_peer_identities, - )?; - mesh_plan.existing_peer_identities = existing_peer_identities.values().cloned().collect(); - write_json(&artifact_root.join("mesh-provision-plan.json"), &mesh_plan)?; - crate::direct_runtime::provision_mesh_filesystem_with_peer_identities( - &mesh_plan, - artifact_root, - &existing_peer_identities, - )?; - for relative_dir in mesh_dirs.values() { - crate::direct_runtime::project_existing_peer_identities_into_mesh_config( - &artifact_root.join(relative_dir).join(MESH_CONFIG_FILENAME), - &existing_peer_identities, - )?; - } - rewrite_compose_mesh_bind_mounts(artifact_root, &mesh_dirs)?; - let overlay_payload = build_dynamic_compose_route_overlay_payload( - artifact_root, - &site_plan.assigned_components, - &component_mesh_dirs, - &site_plan.proxy_exports, - &site_plan.routed_inputs, - existing_site_peer_identities, - )?; - write_json(&dynamic_route_overlay_path(artifact_root), &overlay_payload)?; - write_embedded_compose_mesh_provision_plan(artifact_root, &mesh_plan)?; - - write_json( - &dynamic_compose_child_metadata_path(artifact_root), - &DynamicComposeChildMetadata { - schema: DYNAMIC_COMPOSE_CHILD_SCHEMA.to_string(), - version: DYNAMIC_COMPOSE_CHILD_VERSION, - services: kept_services.iter().cloned().collect(), - readiness_services: root_services - .into_iter() - .filter(|service| kept_services.contains(service)) - .collect(), - }, - ) -} - -fn read_compose_document(path: &Path) -> Result { - let raw = fs::read_to_string(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - serde_yaml::from_str(&raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid compose file {}", path.display())) -} - -fn read_embedded_compose_mesh_provision_plan(artifact_root: &Path) -> Result { - let path = artifact_root.join("compose.yaml"); - let document = read_compose_document(&path)?; - let Some(root) = document.as_mapping() else { - return Err(miette::miette!( - "compose file {} is not a YAML mapping", - path.display() - )); - }; - let configs_key = serde_yaml::Value::String("configs".to_string()); - let config_name = serde_yaml::Value::String("amber-mesh-provision-plan".to_string()); - let content_key = serde_yaml::Value::String("content".to_string()); - let content = root - .get(&configs_key) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|configs| configs.get(&config_name)) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|config| config.get(&content_key)) - .and_then(serde_yaml::Value::as_str) - .ok_or_else(|| { - miette::miette!( - "compose file {} is missing configs.amber-mesh-provision-plan.content", - path.display() - ) - })?; - serde_json::from_str(content).map_err(|err| { - miette::miette!( - "compose file {} has invalid embedded mesh provision plan: {err}", - path.display() - ) - }) -} - -fn write_embedded_compose_mesh_provision_plan( - artifact_root: &Path, - plan: &MeshProvisionPlan, -) -> Result<()> { - let path = artifact_root.join("compose.yaml"); - let mut document = read_compose_document(&path)?; - let plan_json = serde_json::to_string_pretty(plan) - .into_diagnostic() - .wrap_err("failed to serialize compose mesh provision plan")?; - document - .as_mapping_mut() - .and_then(|root| root.get_mut(yaml_string("configs"))) - .and_then(serde_yaml::Value::as_mapping_mut) - .and_then(|configs| configs.get_mut(yaml_string("amber-mesh-provision-plan"))) - .and_then(serde_yaml::Value::as_mapping_mut) - .ok_or_else(|| { - miette::miette!( - "compose file {} is missing configs.amber-mesh-provision-plan", - path.display() - ) - })? - .insert(yaml_string("content"), serde_yaml::Value::String(plan_json)); - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; - fs::write(&path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) -} - -fn read_embedded_kubernetes_mesh_provision_plan(artifact_root: &Path) -> Result { - let path = artifact_root - .join("01-configmaps") - .join("amber-mesh-provision.yaml"); - let raw = fs::read_to_string(&path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let document: serde_yaml::Value = - serde_yaml::from_str(&raw) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "invalid kubernetes mesh provision configmap {}", - path.display() - ) - })?; - let mesh_plan = document - .as_mapping() - .and_then(|root| root.get(yaml_string("data"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|data| data.get(yaml_string("mesh-plan.json"))) - .and_then(serde_yaml::Value::as_str) - .ok_or_else(|| { - miette::miette!( - "kubernetes mesh provision configmap {} is missing data.mesh-plan.json", - path.display() - ) - })?; - serde_json::from_str(mesh_plan).map_err(|err| { - miette::miette!( - "kubernetes mesh provision configmap {} has invalid mesh plan: {err}", - path.display() - ) - }) -} - -fn read_kubernetes_runtime_mesh_provision_plan(artifact_root: &Path) -> Result { - let path = artifact_root.join("mesh-provision-plan.json"); - if path.is_file() { - return read_json(&path, "mesh provision plan"); - } - read_embedded_kubernetes_mesh_provision_plan(artifact_root) -} - -fn write_embedded_kubernetes_mesh_provision_plan( - artifact_root: &Path, - plan: &MeshProvisionPlan, -) -> Result<()> { - let path = artifact_root.join(KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH); - let raw = fs::read_to_string(&path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "invalid kubernetes mesh provision configmap {}", - path.display() - ) - })?; - let plan_json = serde_json::to_string_pretty(plan) - .into_diagnostic() - .wrap_err("failed to serialize kubernetes mesh provision plan")?; - document - .as_mapping_mut() - .and_then(|root| root.get_mut(yaml_string("data"))) - .and_then(serde_yaml::Value::as_mapping_mut) - .ok_or_else(|| { - miette::miette!( - "kubernetes mesh provision configmap {} is missing a data mapping", - path.display() - ) - })? - .insert( - yaml_string("mesh-plan.json"), - serde_yaml::Value::String(plan_json), - ); - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; - fs::write(&path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) -} - -fn kubernetes_resource_name(document: &serde_yaml::Value) -> Option<&str> { - document - .as_mapping() - .and_then(|root| root.get(yaml_string("metadata"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|metadata| metadata.get(yaml_string("name"))) - .and_then(serde_yaml::Value::as_str) -} - -fn kubernetes_dynamic_apply_resource_kept_from_contents( - resource: &str, - raw: &str, - child_component_labels: &BTreeSet, -) -> Result { - if matches!( - resource, - KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH - | KUBERNETES_PROVISIONER_JOB_PATH - | KUBERNETES_PROVISIONER_ROLE_PATH - | KUBERNETES_PROVISIONER_ROLEBINDING_PATH - | KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH - ) || resource.starts_with("03-persistentvolumeclaims/") - { - return Ok(true); - } - - let document: serde_yaml::Value = serde_yaml::from_str(raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid kubernetes resource {resource}"))?; - Ok(document - .as_mapping() - .and_then(|root| root.get(yaml_string("metadata"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|metadata| metadata.get(yaml_string("labels"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|labels| labels.get(yaml_string("amber.io/component-id"))) - .and_then(serde_yaml::Value::as_str) - .is_some_and(|component_id| child_component_labels.contains(component_id))) -} - -pub(crate) fn project_kubernetes_dynamic_child_artifact_files( - artifact_files: &BTreeMap, - component_ids: &[usize], -) -> Result> { - let child_component_labels = component_ids - .iter() - .map(|component_id| format!("c{component_id}")) - .collect::>(); - let kustomization_path = "kustomization.yaml"; - let raw = artifact_files.get(kustomization_path).ok_or_else(|| { - miette::miette!("dynamic kubernetes artifact snapshot is missing {kustomization_path}") - })?; - let mut document: serde_yaml::Value = serde_yaml::from_str(raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid kustomization {kustomization_path}"))?; - let root = document.as_mapping_mut().ok_or_else(|| { - miette::miette!("kustomization {kustomization_path} is not a YAML mapping") - })?; - let resources = root - .get_mut(yaml_string("resources")) - .and_then(serde_yaml::Value::as_sequence_mut) - .ok_or_else(|| { - miette::miette!("kustomization {kustomization_path} is missing a resources sequence") - })?; - let mut projected = artifact_files - .iter() - .filter(|(path, _)| !path.ends_with(".yaml") && path.as_str() != kustomization_path) - .map(|(path, contents)| (path.clone(), contents.clone())) - .collect::>(); - let mut kept_resources = Vec::new(); - let mut kept_resource_names = BTreeSet::new(); - for resource in resources - .iter() - .filter_map(serde_yaml::Value::as_str) - .map(str::to_owned) - { - let raw = artifact_files - .get(&resource) - .ok_or_else(|| miette::miette!("dynamic kubernetes artifact is missing {resource}"))?; - if !kubernetes_dynamic_apply_resource_kept_from_contents( - &resource, - raw, - &child_component_labels, - )? { - continue; - } - let document: serde_yaml::Value = serde_yaml::from_str(raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid kubernetes resource {resource}"))?; - if let Some(name) = kubernetes_resource_name(&document) { - kept_resource_names.insert(name.to_string()); - } - projected.insert(resource.clone(), raw.clone()); - kept_resources.push(serde_yaml::Value::String(resource)); - } - *resources = kept_resources; - - if let Some(generators) = root - .get_mut(yaml_string("secretGenerator")) - .and_then(serde_yaml::Value::as_sequence_mut) - { - generators.retain(|generator| { - generator - .as_mapping() - .and_then(|mapping| mapping.get(yaml_string("name"))) - .and_then(serde_yaml::Value::as_str) - != Some(KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME) - }); - } - - if let Some(replacements) = root - .get_mut(yaml_string("replacements")) - .and_then(serde_yaml::Value::as_sequence_mut) - { - replacements.retain_mut(|replacement| { - let Some(targets) = replacement - .as_mapping_mut() - .and_then(|mapping| mapping.get_mut(yaml_string("targets"))) - .and_then(serde_yaml::Value::as_sequence_mut) - else { - return false; - }; - targets.retain(|target| { - target - .as_mapping() - .and_then(|mapping| mapping.get(yaml_string("select"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|select| select.get(yaml_string("name"))) - .and_then(serde_yaml::Value::as_str) - .is_some_and(|name| kept_resource_names.contains(name)) - }); - !targets.is_empty() - }); - } - - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {kustomization_path}"))?; - projected.insert(kustomization_path.to_string(), rendered); - Ok(projected) -} - -fn project_kubernetes_dynamic_child_destroy_artifact_files( - artifact_files: &BTreeMap, -) -> Result> { - let kustomization_path = "kustomization.yaml"; - let raw = artifact_files.get(kustomization_path).ok_or_else(|| { - miette::miette!("dynamic kubernetes artifact snapshot is missing {kustomization_path}") - })?; - let mut document: serde_yaml::Value = serde_yaml::from_str(raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid kustomization {kustomization_path}"))?; - let root = document.as_mapping_mut().ok_or_else(|| { - miette::miette!("kustomization {kustomization_path} is not a YAML mapping") - })?; - let resources = root - .get_mut(yaml_string("resources")) - .and_then(serde_yaml::Value::as_sequence_mut) - .ok_or_else(|| { - miette::miette!("kustomization {kustomization_path} is missing a resources sequence") - })?; - let mut projected = artifact_files - .iter() - .filter(|(path, _)| !path.ends_with(".yaml") && path.as_str() != kustomization_path) - .map(|(path, contents)| (path.clone(), contents.clone())) - .collect::>(); - let shared_paths = [ - KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH, - KUBERNETES_PROVISIONER_ROLE_PATH, - KUBERNETES_PROVISIONER_ROLEBINDING_PATH, - KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH, - ] - .into_iter() - .collect::>(); - let mut kept_resources = Vec::new(); - for resource in resources - .iter() - .filter_map(serde_yaml::Value::as_str) - .map(str::to_owned) - { - if shared_paths.contains(resource.as_str()) { - continue; - } - let raw = artifact_files - .get(&resource) - .ok_or_else(|| miette::miette!("dynamic kubernetes artifact is missing {resource}"))?; - projected.insert(resource.clone(), raw.clone()); - kept_resources.push(serde_yaml::Value::String(resource)); - } - *resources = kept_resources; - - if let Some(generators) = root - .get_mut(yaml_string("secretGenerator")) - .and_then(serde_yaml::Value::as_sequence_mut) - { - generators.retain(|generator| { - generator - .as_mapping() - .and_then(|mapping| mapping.get(yaml_string("name"))) - .and_then(serde_yaml::Value::as_str) - != Some(KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME) - }); - } - - if let Some(replacements) = root - .get_mut(yaml_string("replacements")) - .and_then(serde_yaml::Value::as_sequence_mut) - { - replacements.clear(); - } - - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {kustomization_path}"))?; - projected.insert(kustomization_path.to_string(), rendered); - Ok(projected) -} - -fn read_artifact_snapshot(root: &Path) -> Result> { - walk_files(root)? - .into_iter() - .map(|path| { - let relative = path - .strip_prefix(root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to relativize {}", path.display()))?; - let relative = path_to_forward_slash_string(relative); - let contents = fs::read_to_string(&path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - Ok((relative, contents)) - }) - .collect() -} - -fn rewrite_dynamic_kubernetes_apply_bundle( - artifact_root: &Path, - component_ids: &[usize], -) -> Result<()> { - let files = read_artifact_snapshot(artifact_root)?; - let projected = project_kubernetes_dynamic_child_artifact_files(&files, component_ids)?; - replace_artifact_snapshot(artifact_root, &projected) -} - -fn prepare_dynamic_kubernetes_child_artifact( - plan: &SiteControllerRuntimePlan, - site_plan: &DynamicSitePlanRecord, - artifact_root: &Path, - existing_site_peer_identities: &BTreeMap, -) -> Result<()> { - project_dynamic_child_mesh_scope(artifact_root, Some(&plan.mesh_scope))?; - let plan_path = artifact_root.join("mesh-provision-plan.json"); - let mesh_plan = read_embedded_kubernetes_mesh_provision_plan(artifact_root)?; - let router_mesh_port = router_mesh_port_from_plan(&mesh_plan, "kubernetes")?; - let assigned = site_plan - .assigned_components - .iter() - .map(String::as_str) - .collect::>(); - let mut kept_component_ids = BTreeSet::new(); - let mut router_target = None; - let mut overlay_targets = Vec::with_capacity(mesh_plan.targets.len()); - let mut provision_targets = Vec::new(); - for target in mesh_plan.targets { - match target.kind { - MeshProvisionTargetKind::Component => { - if assigned.contains(target.config.identity.id.as_str()) { - kept_component_ids.insert(target.config.identity.id.clone()); - provision_targets.push(target.clone()); - overlay_targets.push(target); - } - } - MeshProvisionTargetKind::Router => { - router_target = Some(target); - } - } - } - - let Some(mut router_target) = router_target else { - return Err(miette::miette!( - "dynamic mesh provision plan {} is missing a router target", - plan_path.display() - )); - }; - filter_dynamic_router_target(&mut router_target, &kept_component_ids); - overlay_targets.push(router_target); - let mut overlay_plan = MeshProvisionPlan { - version: mesh_plan.version.clone(), - identity_seed: mesh_plan.identity_seed.clone(), - existing_peer_identities: Vec::new(), - targets: overlay_targets, - }; - let overlay_existing_peer_identities = - crate::direct_runtime::required_existing_mesh_peer_identities( - &overlay_plan, - existing_site_peer_identities, - )?; - overlay_plan.existing_peer_identities = - overlay_existing_peer_identities.values().cloned().collect(); - write_json(&plan_path, &overlay_plan)?; - let mut provision_plan = MeshProvisionPlan { - version: mesh_plan.version, - identity_seed: mesh_plan.identity_seed, - existing_peer_identities: Vec::new(), - targets: provision_targets, - }; - ensure_dynamic_proxy_export_component_routes( - &mut provision_plan, - &site_plan.proxy_exports, - &plan.router_identity_id, - )?; - rewrite_dynamic_routed_inputs( - &mut provision_plan, - &site_plan.routed_inputs, - SiteKind::Kubernetes, - &plan.router_identity_id, - Some(router_mesh_port), - )?; - let provision_existing_peer_identities = - crate::direct_runtime::required_existing_mesh_peer_identities( - &provision_plan, - existing_site_peer_identities, - )?; - provision_plan.existing_peer_identities = provision_existing_peer_identities - .values() - .cloned() - .collect(); - write_embedded_kubernetes_mesh_provision_plan(artifact_root, &provision_plan)?; - project_dynamic_kubernetes_proxy_export_resources( - artifact_root, - &provision_plan, - &site_plan.proxy_exports, - )?; - rewrite_dynamic_kubernetes_apply_bundle(artifact_root, &site_plan.component_ids) -} - -fn kubernetes_peer_addrs_for_artifact(artifact_root: &Path) -> Result> { - let mesh_plan = read_kubernetes_runtime_mesh_provision_plan(artifact_root)?; - mesh_plan - .targets - .iter() - .filter(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - .map(|target| { - Ok(( - target.config.identity.id.clone(), - kubernetes_component_mesh_peer_addr( - artifact_root, - &target.config.identity.id, - &target.output, - target.config.mesh_listen.port(), - )?, - )) - }) - .collect() -} - -fn build_kubernetes_route_overlay_base( - artifact_root: &Path, - assigned_components: &[String], - provider_peer_addrs: &BTreeMap, - peer_identities: &BTreeMap, -) -> Result { - let mesh_plan = read_kubernetes_runtime_mesh_provision_plan(artifact_root)?; - let kept_component_ids = assigned_components.iter().cloned().collect::>(); - let mut router_target = mesh_plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - .cloned() - .ok_or_else(|| { - miette::miette!( - "kubernetes artifact {} is missing a router mesh target", - artifact_root.display() - ) - })?; - filter_dynamic_router_target(&mut router_target, &kept_component_ids); - for route in &mut router_target.config.inbound { - if let InboundTarget::MeshForward { - peer_id, peer_addr, .. - } = &mut route.target - && let Some(resolved) = provider_peer_addrs.get(peer_id) - { - *peer_addr = resolved.clone(); - } - } - let peers = router_target - .config - .peers - .iter() - .map(|peer| { - peer_identities.get(&peer.id).map(|identity| MeshPeer { - id: identity.id.clone(), - public_key: identity.public_key, - }) - }) - .collect::>>() - .ok_or_else(|| { - miette::miette!( - "kubernetes artifact {} is missing a live mesh identity for one of its route peers", - artifact_root.display() - ) - })?; - Ok(StoredRouteOverlayPayload { - peers, - inbound_routes: router_target.config.inbound, - }) -} - -fn write_kubernetes_live_route_overlay_payload( - artifact_root: &Path, - assigned_components: &[String], - proxy_exports: &BTreeMap, - routed_inputs: &[DynamicInputRouteRecord], - peer_identities: &BTreeMap, -) -> Result<()> { - let provider_peer_addrs = kubernetes_peer_addrs_for_artifact(artifact_root)?; - let mut payload = build_kubernetes_route_overlay_base( - artifact_root, - assigned_components, - &provider_peer_addrs, - peer_identities, - )?; - augment_route_overlay_payload( - &mut payload, - proxy_exports, - routed_inputs, - &provider_peer_addrs, - peer_identities, - None, - false, - )?; - write_dynamic_route_overlay_payload(artifact_root, &payload) -} - -fn kubernetes_network_policy_paths_by_component_label( - artifact_root: &Path, -) -> Result> { - let netpol_root = artifact_root.join("05-networkpolicies"); - if !netpol_root.is_dir() { - return Ok(BTreeMap::new()); - } - let mut netpol_paths = BTreeMap::new(); - for path in walk_files(&netpol_root)? { - let raw = fs::read_to_string(&path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let document: serde_yaml::Value = serde_yaml::from_str(&raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid kubernetes network policy {}", path.display()))?; - let Some(root) = document.as_mapping() else { - continue; - }; - if root - .get(yaml_string("kind")) - .and_then(serde_yaml::Value::as_str) - != Some("NetworkPolicy") - { - continue; - } - let Some(component_label) = root - .get(yaml_string("metadata")) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|metadata| metadata.get(yaml_string("labels"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|labels| labels.get(yaml_string("amber.io/component-id"))) - .and_then(serde_yaml::Value::as_str) - else { - continue; - }; - netpol_paths.insert(component_label.to_string(), path); - } - Ok(netpol_paths) -} - -fn project_dynamic_kubernetes_proxy_export_resources( - artifact_root: &Path, - mesh_plan: &MeshProvisionPlan, - proxy_exports: &BTreeMap, -) -> Result<()> { - if proxy_exports.is_empty() { - return Ok(()); - } - - let netpol_paths = kubernetes_network_policy_paths_by_component_label(artifact_root)?; - let exported_mesh_ports = proxy_exports - .values() - .map(|export| { - let component_target = mesh_plan - .targets - .iter() - .find(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && target.config.identity.id == export.component - }) - .ok_or_else(|| { - miette::miette!( - "dynamic proxy export provider {} is missing from the kubernetes mesh plan", - export.component - ) - })?; - Ok(( - format!("c{}", export.component_id), - component_target.config.mesh_listen.port(), - )) - }) - .collect::>>()?; - - for (component_label, mesh_port) in exported_mesh_ports { - let path = netpol_paths.get(&component_label).ok_or_else(|| { - miette::miette!( - "dynamic proxy export provider {component_label} is missing a kubernetes network \ - policy in {}", - artifact_root.join("05-networkpolicies").display() - ) - })?; - ensure_kubernetes_network_policy_router_ingress(path, mesh_port)?; - } - - Ok(()) -} - -fn ensure_kubernetes_network_policy_router_ingress(path: &Path, mesh_port: u16) -> Result<()> { - let raw = fs::read_to_string(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid kubernetes network policy {}", path.display()))?; - let root = document.as_mapping_mut().ok_or_else(|| { - miette::miette!( - "kubernetes network policy {} is not a YAML mapping", - path.display() - ) - })?; - let spec = root - .get_mut(yaml_string("spec")) - .and_then(serde_yaml::Value::as_mapping_mut) - .ok_or_else(|| { - miette::miette!( - "kubernetes network policy {} is missing a spec mapping", - path.display() - ) - })?; - let ingress = spec - .entry(yaml_string("ingress")) - .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) - .as_sequence_mut() - .ok_or_else(|| { - miette::miette!( - "kubernetes network policy {} has non-sequence spec.ingress", - path.display() - ) - })?; - - let router_peer = kubernetes_router_network_policy_peer(); - let mut matched_port_rule = false; - let mut changed = false; - for rule in ingress.iter_mut() { - let Some(rule_mapping) = rule.as_mapping_mut() else { - continue; - }; - let matches_port = rule_mapping - .get(yaml_string("ports")) - .and_then(serde_yaml::Value::as_sequence) - .is_some_and(|ports| { - ports - .iter() - .any(|port| network_policy_port_matches(port, mesh_port)) - }); - if !matches_port { - continue; - } - matched_port_rule = true; - let Some(from) = rule_mapping - .get_mut(yaml_string("from")) - .and_then(serde_yaml::Value::as_sequence_mut) - else { - break; - }; - if from.iter().any(network_policy_peer_is_router) { - break; - } - from.push(router_peer.clone()); - changed = true; - break; - } - - if !matched_port_rule { - ingress.push(serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter( - [ - ( - yaml_string("from"), - serde_yaml::Value::Sequence(vec![router_peer.clone()]), - ), - ( - yaml_string("ports"), - serde_yaml::Value::Sequence(vec![serde_yaml::Value::Mapping( - serde_yaml::Mapping::from_iter([ - (yaml_string("protocol"), yaml_string("TCP")), - ( - yaml_string("port"), - serde_yaml::Value::Number(u64::from(mesh_port).into()), - ), - ]), - )]), - ), - ], - ))); - changed = true; - } - - if !changed { - return Ok(()); - } - - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; - fs::write(path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) -} - -fn kubernetes_router_network_policy_peer() -> serde_yaml::Value { - serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( - yaml_string("podSelector"), - serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( - yaml_string("matchLabels"), - serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( - yaml_string("amber.io/component"), - yaml_string(KUBERNETES_ROUTER_COMPONENT_NAME), - )])), - )])), - )])) -} - -fn network_policy_port_matches(port: &serde_yaml::Value, expected_port: u16) -> bool { - port.as_mapping() - .and_then(|port| port.get(yaml_string("port"))) - .and_then(|value| { - value - .as_u64() - .or_else(|| value.as_i64().and_then(|value| u64::try_from(value).ok())) - }) - == Some(u64::from(expected_port)) -} - -fn network_policy_peer_is_router(peer: &serde_yaml::Value) -> bool { - peer.as_mapping() - .and_then(|peer| peer.get(yaml_string("podSelector"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|selector| selector.get(yaml_string("matchLabels"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|labels| labels.get(yaml_string("amber.io/component"))) - .and_then(serde_yaml::Value::as_str) - == Some(KUBERNETES_ROUTER_COMPONENT_NAME) -} - -fn dynamic_proxy_export_kubernetes_peer_addr( - artifact_root: &Path, - mesh_plan: &MeshProvisionPlan, - export: &DynamicProxyExportRecord, -) -> Result { - let component_target = mesh_plan - .targets - .iter() - .find(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && target.config.identity.id == export.component - }) - .ok_or_else(|| { - miette::miette!( - "dynamic proxy export provider {} is missing from the kubernetes mesh plan", - export.component - ) - })?; - kubernetes_component_mesh_peer_addr( - artifact_root, - &export.component, - &component_target.output, - component_target.config.mesh_listen.port(), - ) -} - -fn project_dynamic_child_mesh_scope(artifact_root: &Path, mesh_scope: Option<&str>) -> Result<()> { - let Some(mesh_scope) = mesh_scope else { - return Ok(()); - }; - let path = artifact_root.join("mesh-provision-plan.json"); - if path.is_file() { - let mut plan: MeshProvisionPlan = read_json(&path, "mesh provision plan")?; - let existing_scopes = mesh_provision_plan_scopes(&plan); - if !project_mesh_provision_plan_scope(&mut plan, mesh_scope) { - return Ok(()); - } - write_json(&path, &plan)?; - return rewrite_dynamic_artifact_mesh_scope_literals( - artifact_root, - &existing_scopes, - mesh_scope, - ); - } - - let compose_path = artifact_root.join("compose.yaml"); - if compose_path.is_file() { - let mut plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; - let existing_scopes = mesh_provision_plan_scopes(&plan); - if !project_mesh_provision_plan_scope(&mut plan, mesh_scope) { - return Ok(()); - } - write_embedded_compose_mesh_provision_plan(artifact_root, &plan)?; - return rewrite_dynamic_artifact_mesh_scope_literals( - artifact_root, - &existing_scopes, - mesh_scope, - ); - } - - let configmap_path = artifact_root.join(KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH); - if configmap_path.is_file() { - let mut plan = read_embedded_kubernetes_mesh_provision_plan(artifact_root)?; - let existing_scopes = mesh_provision_plan_scopes(&plan); - if !project_mesh_provision_plan_scope(&mut plan, mesh_scope) { - return Ok(()); - } - write_embedded_kubernetes_mesh_provision_plan(artifact_root, &plan)?; - return rewrite_dynamic_artifact_mesh_scope_literals( - artifact_root, - &existing_scopes, - mesh_scope, - ); - } - - Err(miette::miette!( - "dynamic artifact {} is missing a mesh provision plan", - artifact_root.display() - )) -} - -fn project_mesh_provision_plan_scope(plan: &mut MeshProvisionPlan, mesh_scope: &str) -> bool { - let mut changed = false; - for target in &mut plan.targets { - if target.config.identity.mesh_scope.as_deref() == Some(mesh_scope) { - continue; - } - target.config.identity.mesh_scope = Some(mesh_scope.to_string()); - changed = true; - } - changed -} - -fn mesh_provision_plan_scopes(plan: &MeshProvisionPlan) -> BTreeSet { - let mut scopes = BTreeSet::new(); - for target in &plan.targets { - if let Some(scope) = target.config.identity.mesh_scope.as_deref() { - scopes.insert(scope.to_string()); - } - } - for identity in &plan.existing_peer_identities { - if let Some(scope) = identity.mesh_scope.as_deref() { - scopes.insert(scope.to_string()); - } - } - scopes -} - -fn rewrite_dynamic_artifact_mesh_scope_literals( - artifact_root: &Path, - existing_scopes: &BTreeSet, - mesh_scope: &str, -) -> Result<()> { - let rewrite_scopes = existing_scopes - .iter() - .filter(|scope| scope.as_str() != mesh_scope) - .cloned() - .collect::>(); - if rewrite_scopes.is_empty() { - return Ok(()); - } - - for path in walk_files(artifact_root)? { - match path.extension().and_then(|extension| extension.to_str()) { - Some("json") => rewrite_json_scope_literals(&path, &rewrite_scopes, mesh_scope)?, - Some("yaml" | "yml") => { - rewrite_yaml_scope_literals(&path, &rewrite_scopes, mesh_scope)? - } - Some("env") => rewrite_env_scope_literals(&path, &rewrite_scopes, mesh_scope)?, - _ => {} - } - } - Ok(()) -} - -fn rewrite_json_scope_literals( - path: &Path, - rewrite_scopes: &BTreeSet, - mesh_scope: &str, -) -> Result<()> { - let raw = fs::read_to_string(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let mut document: serde_json::Value = serde_json::from_str(&raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid json {}", path.display()))?; - if !rewrite_scope_json_value(&mut document, rewrite_scopes, mesh_scope) { - return Ok(()); - } - let rendered = serde_json::to_string_pretty(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; - fs::write(path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) -} - -fn rewrite_yaml_scope_literals( - path: &Path, - rewrite_scopes: &BTreeSet, - mesh_scope: &str, -) -> Result<()> { - let raw = fs::read_to_string(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid yaml {}", path.display()))?; - if !rewrite_scope_yaml_value(&mut document, rewrite_scopes, mesh_scope) { - return Ok(()); - } - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; - fs::write(path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) -} - -fn rewrite_env_scope_literals( - path: &Path, - rewrite_scopes: &BTreeSet, - mesh_scope: &str, -) -> Result<()> { - let raw = fs::read_to_string(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - let mut changed = false; - let mut rendered = raw - .lines() - .map(|line| { - let Some((name, value)) = line.split_once('=') else { - return line.to_string(); - }; - if !rewrite_scopes.contains(value) { - return line.to_string(); - } - changed = true; - format!("{name}={mesh_scope}") - }) - .collect::>() - .join("\n"); - if !changed { - return Ok(()); - } - if raw.ends_with('\n') { - rendered.push('\n'); - } - fs::write(path, rendered) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) -} - -fn rewrite_scope_json_value( - value: &mut serde_json::Value, - rewrite_scopes: &BTreeSet, - mesh_scope: &str, -) -> bool { - match value { - serde_json::Value::String(string) => { - rewrite_scope_string_value(string, rewrite_scopes, mesh_scope) - } - serde_json::Value::Array(values) => { - let mut changed = false; - for value in values { - changed |= rewrite_scope_json_value(value, rewrite_scopes, mesh_scope); - } - changed - } - serde_json::Value::Object(map) => { - let mut changed = false; - for value in map.values_mut() { - changed |= rewrite_scope_json_value(value, rewrite_scopes, mesh_scope); - } - changed - } - _ => false, - } -} - -fn rewrite_scope_yaml_value( - value: &mut serde_yaml::Value, - rewrite_scopes: &BTreeSet, - mesh_scope: &str, -) -> bool { - match value { - serde_yaml::Value::String(string) => { - rewrite_scope_string_value(string, rewrite_scopes, mesh_scope) - } - serde_yaml::Value::Sequence(values) => { - let mut changed = false; - for value in values { - changed |= rewrite_scope_yaml_value(value, rewrite_scopes, mesh_scope); - } - changed - } - serde_yaml::Value::Mapping(map) => { - let mut changed = false; - for (_, value) in map.iter_mut() { - changed |= rewrite_scope_yaml_value(value, rewrite_scopes, mesh_scope); - } - changed - } - _ => false, - } -} - -fn rewrite_scope_string_value( - string: &mut String, - rewrite_scopes: &BTreeSet, - mesh_scope: &str, -) -> bool { - if rewrite_scopes.contains(string) { - *string = mesh_scope.to_string(); - return true; - } - let Some((name, value)) = string.split_once('=') else { - return false; - }; - if !rewrite_scopes.contains(value) { - return false; - } - *string = format!("{name}={mesh_scope}"); - true -} - -fn reconcile_artifact_files(site_plan: &DynamicSitePlanRecord) -> &BTreeMap { - if site_plan.desired_artifact_files.is_empty() { - &site_plan.artifact_files - } else { - &site_plan.desired_artifact_files - } -} - -fn load_kubernetes_mesh_secret_payload( - plan: &SiteControllerRuntimePlan, - name: &str, - namespace: Option<&str>, -) -> Result<(String, KubernetesSecretPayload)> { - let namespace = namespace - .or(plan.kubernetes_namespace.as_deref()) - .ok_or_else(|| { - miette::miette!( - "kubernetes site `{}` is missing its namespace", - plan.site_id - ) - })? - .to_string(); - let output = kubectl_command(plan.context.as_deref()) - .arg("-n") - .arg(&namespace) - .arg("get") - .arg("secret") - .arg(name) - .arg("-o") - .arg("json") - .output() - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to read kubernetes mesh secret {} in namespace {}", - name, namespace - ) - })?; - if !output.status.success() { - return Err(miette::miette!( - "failed to read kubernetes mesh secret {} in namespace {}: {}", - name, - namespace, - String::from_utf8_lossy(&output.stderr).trim() - )); - } - let payload: KubernetesSecretPayload = - serde_json::from_slice(&output.stdout).map_err(|err| { - miette::miette!( - "invalid kubernetes secret payload for {} in namespace {}: {err}", - name, - namespace - ) - })?; - Ok((namespace, payload)) -} - -fn decode_kubernetes_mesh_secret_json( - payload: &KubernetesSecretPayload, - namespace: &str, - name: &str, - key: &str, - description: &str, -) -> Result -where - T: for<'de> Deserialize<'de>, -{ - let encoded = payload.data.get(key).ok_or_else(|| { - miette::miette!( - "kubernetes mesh secret {} in namespace {} is missing {}", - name, - namespace, - key - ) - })?; - let bytes = base64::engine::general_purpose::STANDARD - .decode(encoded.as_bytes()) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to decode kubernetes {description} {} in namespace {}", - name, namespace - ) - })?; - serde_json::from_slice(&bytes).map_err(|err| { - miette::miette!( - "invalid kubernetes {description} {} in namespace {}: {err}", - name, - namespace - ) - }) -} - -fn load_kubernetes_mesh_identity_secret( - plan: &SiteControllerRuntimePlan, - name: &str, - namespace: Option<&str>, -) -> Result { - let (namespace, payload) = load_kubernetes_mesh_secret_payload(plan, name, namespace)?; - decode_kubernetes_mesh_secret_json( - &payload, - &namespace, - name, - MESH_IDENTITY_FILENAME, - "mesh identity", - ) -} - -fn load_kubernetes_mesh_config_public( - plan: &SiteControllerRuntimePlan, - name: &str, - namespace: Option<&str>, -) -> Result { - let (namespace, payload) = load_kubernetes_mesh_secret_payload(plan, name, namespace)?; - decode_kubernetes_mesh_secret_json( - &payload, - &namespace, - name, - MESH_CONFIG_FILENAME, - "mesh config", - ) -} - -fn reconcile_site_proxy_metadata( - site_artifact_root: &Path, - site_plan: &DynamicSitePlanRecord, -) -> Result<()> { - let Some(proxy_metadata) = reconcile_artifact_files(site_plan).get("amber-proxy.json") else { - return Ok(()); - }; - let path = site_artifact_root.join("amber-proxy.json"); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - fs::write(&path, proxy_metadata) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display())) -} - -fn project_dynamic_direct_router_surface( - plan: &SiteControllerRuntimePlan, - child: &SiteControllerRuntimeChildRecord, -) -> Result<()> { - let state_path = direct_runtime_state_path(Path::new(&child.artifact_root)); - let router_mesh_port = plan.router_mesh_port.ok_or_else(|| { - miette::miette!( - "direct site `{}` is missing its router mesh port", - plan.site_id - ) - })?; - let deadline = std::time::Instant::now() + Duration::from_secs(5); - loop { - let mut state: crate::direct_runtime::DirectRuntimeState = - read_json(&state_path, "direct runtime state")?; - if state.router_mesh_port == Some(router_mesh_port) { - break; - } - state.router_mesh_port = Some(router_mesh_port); - write_json(&state_path, &state)?; - std::thread::sleep(Duration::from_millis(100)); - let state: crate::direct_runtime::DirectRuntimeState = - read_json(&state_path, "direct runtime state")?; - if state.router_mesh_port == Some(router_mesh_port) { - break; - } - if std::time::Instant::now() >= deadline { - return Err(miette::miette!( - "timed out projecting direct child router mesh port into {}", - state_path.display() - )); - } - } - - let site_current = direct_current_control_socket_path(Path::new(&plan.artifact_dir)); - let child_current = direct_current_control_socket_path(Path::new(&child.artifact_root)); - if let Some(parent) = child_current.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - ensure_direct_control_socket_link( - &child_current, - &site_current, - "dynamic child router control symlink", - )?; - let artifact_link = Path::new(&child.artifact_root) - .join(".amber") - .join("control") - .join("router-control.sock"); - if let Some(parent) = artifact_link.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - ensure_direct_control_socket_link( - &artifact_link, - &child_current, - "dynamic child router control artifact symlink", - ) -} - -fn project_dynamic_vm_router_surface( - plan: &SiteControllerRuntimePlan, - child: &SiteControllerRuntimeChildRecord, -) -> Result<()> { - let state_path = Path::new(&child.artifact_root) - .join(".amber") - .join("vm-runtime.json"); - let runtime_root = site_controller_runtime_child_runtime_root(plan, child.child_id); - let router_mesh_port = plan.router_mesh_port.ok_or_else(|| { - miette::miette!("vm site `{}` is missing its router mesh port", plan.site_id) - })?; - let deadline = std::time::Instant::now() + Duration::from_secs(5); - loop { - let mut state = - load_vm_runtime_state_for_artifact(Path::new(&child.artifact_root), &runtime_root)?; - if state.router_mesh_port == Some(router_mesh_port) { - break; - } - state.router_mesh_port = Some(router_mesh_port); - write_vm_runtime_state(Path::new(&child.artifact_root), &state)?; - std::thread::sleep(Duration::from_millis(100)); - let state = - load_vm_runtime_state_for_artifact(Path::new(&child.artifact_root), &runtime_root)?; - if state.router_mesh_port == Some(router_mesh_port) { - break; - } - if std::time::Instant::now() >= deadline { - return Err(miette::miette!( - "timed out projecting vm child router mesh port into {}", - state_path.display() - )); - } - } - - let site_current = vm_current_control_socket_path(Path::new(&plan.artifact_dir)); - let child_current = vm_current_control_socket_path(Path::new(&child.artifact_root)); - if let Some(parent) = child_current.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - ensure_control_socket_link( - &child_current, - &site_current, - "dynamic child vm router control symlink", - )?; - let artifact_link = Path::new(&child.artifact_root) - .join(".amber") - .join("control") - .join("router-control.sock"); - if let Some(parent) = artifact_link.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - ensure_control_socket_link( - &artifact_link, - &child_current, - "dynamic child vm router control artifact symlink", - ) -} - -fn dynamic_child_route_overlay_id(plan: &SiteControllerRuntimePlan, child_id: u64) -> String { - format!("framework-child:{}:{child_id}", plan.site_id) -} - -fn site_router_control_endpoint(plan: &SiteControllerRuntimePlan) -> Result { - let state_path = Path::new(&plan.site_state_root).join("manager-state.json"); - if state_path.is_file() { - let state: SiteManagerState = read_json(&state_path, "site manager state")?; - if let Some(raw) = state.router_control { - return parse_control_endpoint(&raw); - } - } - - match plan.kind { - SiteKind::Direct => Ok(ControlEndpoint::Unix(direct_current_control_socket_path( - Path::new(&plan.artifact_dir), - ))), - SiteKind::Vm => Ok(ControlEndpoint::Unix(vm_current_control_socket_path( - Path::new(&plan.artifact_dir), - ))), - SiteKind::Compose | SiteKind::Kubernetes => Err(miette::miette!( - "site `{}` manager state is missing router control endpoint", - plan.site_id - )), - } -} - -fn child_router_overlay_payload( - plan: &SiteControllerRuntimePlan, - artifact_root: &Path, - runtime_root: &Path, -) -> Result<(Vec, Vec)> { - let overlay_path = dynamic_route_overlay_path(artifact_root); - if overlay_path.is_file() { - let payload: StoredRouteOverlayPayload = read_json(&overlay_path, "site router overlay")?; - return Ok((payload.peers, payload.inbound_routes)); - } - let provision: MeshProvisionPlan = read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - )?; - let Some(router_target) = provision - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - else { - return Ok((Vec::new(), Vec::new())); - }; - let proxy_exports = load_dynamic_proxy_exports_metadata(artifact_root)?; - let mut overlay_peer_ids = router_target - .config - .peers - .iter() - .map(|peer| peer.id.clone()) - .collect::>(); - overlay_peer_ids.extend( - proxy_exports - .values() - .map(|export| export.component.clone()), - ); - - let mut peers = Vec::new(); - let mut peer_addr_by_id = BTreeMap::new(); - for peer_id in &overlay_peer_ids { - let component_target = provision - .targets - .iter() - .find(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && target.config.identity.id == *peer_id - }) - .ok_or_else(|| { - miette::miette!( - "router overlay peer {} is missing from mesh provision plan", - peer_id - ) - })?; - let (identity, runtime_config) = match &component_target.output { - MeshProvisionOutput::Filesystem { dir } => { - let identity: MeshIdentitySecret = read_json( - &runtime_root.join(dir).join(MESH_IDENTITY_FILENAME), - "mesh identity", - )?; - let runtime_config: MeshConfigPublic = read_json( - &runtime_root.join(dir).join(MESH_CONFIG_FILENAME), - "mesh config", - )?; - (identity, Some(runtime_config)) - } - MeshProvisionOutput::KubernetesSecret { name, namespace } => ( - load_kubernetes_mesh_identity_secret(plan, name, namespace.as_deref())?, - None, - ), - }; - let public_key = identity.public_key().into_diagnostic()?; - if let Some(runtime_config) = runtime_config { - peer_addr_by_id.insert( - runtime_config.identity.id.clone(), - runtime_config.mesh_listen, - ); - } - peers.push(MeshPeer { - id: identity.id, - public_key, - }); - } - let mut inbound_routes = router_target.config.inbound.clone(); - for route in &mut inbound_routes { - if let InboundTarget::MeshForward { - peer_id, peer_addr, .. - } = &mut route.target - && let Some(resolved) = peer_addr_by_id.get(peer_id) - { - *peer_addr = resolved.to_string(); - } - } - if !proxy_exports.is_empty() { - add_dynamic_proxy_export_overlay_routes(&mut inbound_routes, &proxy_exports, |export| { - match plan.kind { - SiteKind::Kubernetes => { - dynamic_proxy_export_kubernetes_peer_addr(artifact_root, &provision, export) - } - SiteKind::Direct | SiteKind::Vm | SiteKind::Compose => peer_addr_by_id - .get(&export.component) - .map(ToString::to_string) - .ok_or_else(|| { - miette::miette!( - "dynamic proxy export provider {} is missing a live mesh address", - export.component - ) - }), - } - })?; - } - Ok((peers, inbound_routes)) -} - -fn child_overlay_runtime_root( - plan: &SiteControllerRuntimePlan, - child: &SiteControllerRuntimeChildRecord, -) -> PathBuf { - match plan.kind { - SiteKind::Direct | SiteKind::Vm => { - site_controller_runtime_child_runtime_root(plan, child.child_id) - } - SiteKind::Compose | SiteKind::Kubernetes => PathBuf::from(&child.artifact_root), - } -} - -fn dynamic_route_issuer_grants( - children: &[SiteControllerRuntimeChildRecord], -) -> Result>> { - let mut issuers_by_route_id = BTreeMap::>::new(); - for child in children { - for input in &child.routed_inputs { - let route_id = - dynamic_input_route_route_id(input, dynamic_input_route_mesh_protocol(input)?); - issuers_by_route_id - .entry(route_id) - .or_default() - .insert(input.component.clone()); - } - } - Ok(issuers_by_route_id) -} - -fn load_published_component_peers( - plan: &SiteControllerRuntimePlan, - published_children: &[SiteControllerRuntimeChildRecord], -) -> Result> { - let mut component_peers = BTreeMap::new(); - for child in published_children { - if child.assigned_components.is_empty() { - continue; - } - let artifact_root = Path::new(&child.artifact_root); - let runtime_root = child_overlay_runtime_root(plan, child); - let provision: MeshProvisionPlan = read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - )?; - for component in &child.assigned_components { - let target = provision - .targets - .iter() - .find(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && target.config.identity.id == *component - }) - .ok_or_else(|| { - miette::miette!( - "published child {} is missing component {} in its mesh provision plan", - child.child_id, - component - ) - })?; - let identity = match &target.output { - MeshProvisionOutput::Filesystem { dir } => read_json( - &runtime_root.join(dir).join(MESH_IDENTITY_FILENAME), - "mesh identity", - )?, - MeshProvisionOutput::KubernetesSecret { name, namespace } => { - load_kubernetes_mesh_identity_secret(plan, name, namespace.as_deref())? - } - }; - component_peers.insert( - component.clone(), - MeshPeer { - id: identity.id.clone(), - public_key: identity.public_key().into_diagnostic()?, - }, - ); - } - } - Ok(component_peers) -} - -fn apply_dynamic_route_issuer_grants( - peers: &mut Vec, - inbound_routes: &mut [InboundRoute], - issuers_by_route_id: &BTreeMap>, - component_peers: &BTreeMap, -) -> Result<()> { - let mut known_peer_ids = peers - .iter() - .map(|peer| peer.id.clone()) - .collect::>(); - for route in inbound_routes { - let Some(issuers) = issuers_by_route_id.get(&route.route_id) else { - continue; - }; - route.allowed_issuers = issuers.iter().cloned().collect(); - for issuer in issuers { - if known_peer_ids.contains(issuer) { - continue; - } - let peer = component_peers.get(issuer).ok_or_else(|| { - miette::miette!( - "dynamic route {} references published issuer {} with no live mesh peer", - route.route_id, - issuer - ) - })?; - peers.push(peer.clone()); - known_peer_ids.insert(issuer.clone()); - } - } - Ok(()) -} - -async fn reconcile_dynamic_site_router_overlays(app: &SiteControllerRuntimeApp) -> Result<()> { - let published_children = { - let state = app.state.lock().await; - state - .children - .values() - .filter(|child| child.published) - .cloned() - .collect::>() - }; - reconcile_dynamic_site_router_overlays_for_children( - app, - &published_children, - &published_children, - ) - .await -} - -async fn reconcile_dynamic_site_router_overlays_for_children( - app: &SiteControllerRuntimeApp, - overlay_children: &[SiteControllerRuntimeChildRecord], - issuer_children: &[SiteControllerRuntimeChildRecord], -) -> Result<()> { - if overlay_children.is_empty() { - return Ok(()); - } - - let issuers_by_route_id = dynamic_route_issuer_grants(issuer_children)?; - let component_peers = load_published_component_peers(&app.plan, issuer_children)?; - for child in overlay_children { - let artifact_root = Path::new(&child.artifact_root); - let runtime_root = child_overlay_runtime_root(&app.plan, child); - let (mut peers, mut inbound_routes) = - child_router_overlay_payload(&app.plan, artifact_root, &runtime_root)?; - apply_dynamic_route_issuer_grants( - &mut peers, - &mut inbound_routes, - &issuers_by_route_id, - &component_peers, - )?; - if inbound_routes.is_empty() { - continue; - } - let endpoint = site_router_control_endpoint(&app.plan)?; - apply_route_overlay_with_retry( - &endpoint, - &dynamic_child_route_overlay_id(&app.plan, child.child_id), - &peers, - &inbound_routes, - Duration::from_secs(30), - ) - .await?; - } - - Ok(()) -} - -async fn apply_dynamic_site_router_overlay( - plan: &SiteControllerRuntimePlan, - child: &SiteControllerRuntimeChildRecord, -) -> Result<()> { - let artifact_root = Path::new(&child.artifact_root); - let runtime_root = child_overlay_runtime_root(plan, child); - let (peers, inbound_routes) = child_router_overlay_payload(plan, artifact_root, &runtime_root)?; - if inbound_routes.is_empty() { - return Ok(()); - } - let endpoint = site_router_control_endpoint(plan)?; - apply_route_overlay_with_retry( - &endpoint, - &dynamic_child_route_overlay_id(plan, child.child_id), - &peers, - &inbound_routes, - Duration::from_secs(30), - ) - .await -} - -async fn revoke_dynamic_site_router_overlay( - plan: &SiteControllerRuntimePlan, - child: &SiteControllerRuntimeChildRecord, -) -> Result<()> { - let endpoint = site_router_control_endpoint(plan)?; - revoke_route_overlay_with_retry( - &endpoint, - &dynamic_child_route_overlay_id(plan, child.child_id), - Duration::from_secs(30), - ) - .await -} - -async fn site_controller_runtime_prepare_child( - app: &SiteControllerRuntimeApp, - child_id: u64, - site_plan: DynamicSitePlanRecord, -) -> Result<()> { - let artifact_root = site_controller_runtime_child_artifact_root(&app.plan, child_id); - let published_children = { - let state = app.state.lock().await; - state - .children - .values() - .filter(|child| child.published) - .cloned() - .collect::>() - }; - replace_artifact_snapshot(&artifact_root, &site_plan.artifact_files)?; - project_dynamic_child_mesh_scope(&artifact_root, Some(&app.plan.mesh_scope))?; - if site_plan.kind == SiteKind::Kubernetes { - let _ = prepare_kubernetes_artifact_namespace( - &app.plan.run_id, - &app.plan.site_id, - &artifact_root, - )?; - } - patch_site_artifacts( - &artifact_root, - &app.plan.run_id, - &app.plan.site_id, - site_plan.kind, - &app.plan.launch_env, - app.plan.observability_endpoint.as_deref(), - )?; - match site_plan.kind { - SiteKind::Direct => { - filter_direct_stage_plan(&artifact_root, &site_plan.component_ids)?; - ensure_dynamic_proxy_export_component_routes_in_artifact( - &artifact_root, - &site_plan.proxy_exports, - &app.plan.router_identity_id, - )?; - rewrite_dynamic_routed_inputs_in_artifact( - &artifact_root, - &site_plan, - app.plan.router_mesh_port, - )?; - write_direct_vm_startup_route_overlay_payload( - &artifact_root, - "direct", - &site_plan.routed_inputs, - &overlay_peer_addr_map_from_ports(&local_direct_peer_ports_for_children( - &app.plan, - &published_children, - )?), - &local_direct_peer_identities_for_children(&app.plan, &published_children)?, - )?; - } - SiteKind::Vm => { - filter_vm_stage_plan(&artifact_root, &site_plan.component_ids)?; - ensure_dynamic_proxy_export_component_routes_in_artifact( - &artifact_root, - &site_plan.proxy_exports, - &app.plan.router_identity_id, - )?; - rewrite_dynamic_routed_inputs_in_artifact( - &artifact_root, - &site_plan, - app.plan.router_mesh_port, - )?; - write_direct_vm_startup_route_overlay_payload( - &artifact_root, - "vm", - &site_plan.routed_inputs, - &overlay_peer_addr_map_from_ports(&local_vm_peer_ports_for_children( - &app.plan, - &published_children, - )?), - &local_vm_peer_identities_for_children(&app.plan, &published_children)?, - )?; - } - SiteKind::Compose => { - let existing_peer_identities = - local_compose_peer_identities(&app.plan, &published_children)?; - prepare_dynamic_compose_child_artifact( - &app.plan, - &site_plan, - &artifact_root, - &published_children, - &existing_peer_identities, - )? - } - SiteKind::Kubernetes => { - let existing_peer_identities = - local_kubernetes_peer_identities(&app.plan, &published_children)?; - prepare_dynamic_kubernetes_child_artifact( - &app.plan, - &site_plan, - &artifact_root, - &existing_peer_identities, - )? - } - } - write_dynamic_proxy_exports_metadata(&artifact_root, &site_plan.proxy_exports)?; - rewrite_dynamic_proxy_metadata(&artifact_root, &site_plan)?; - if site_plan.kind == SiteKind::Compose - && let Some(compose_project) = app.plan.compose_project.as_deref() - { - rewrite_dynamic_compose_proxy_metadata(&artifact_root, compose_project)?; - } - let mut state = app.state.lock().await; - state.children.insert( - child_id, - SiteControllerRuntimeChildRecord { - child_id, - artifact_root: artifact_root.display().to_string(), - assigned_components: site_plan.assigned_components.clone(), - proxy_exports: site_plan.proxy_exports.clone(), - routed_inputs: site_plan.routed_inputs.clone(), - process_pid: None, - published: false, - }, - ); - write_json(&app.state_path, &*state) -} - -async fn site_controller_runtime_publish_child( - app: &SiteControllerRuntimeApp, - child_id: u64, - site_plan: DynamicSitePlanRecord, -) -> Result<()> { - let (child, published_children) = { - let state = app.state.lock().await; - let child = state.children.get(&child_id).cloned().ok_or_else(|| { - miette::miette!("site controller runtime child {child_id} is not prepared") - })?; - let published_children = state - .children - .values() - .filter(|child| child.published) - .cloned() - .collect::>(); - (child, published_children) - }; - if child.published { - return Ok(()); - } - - match app.plan.kind { - SiteKind::Direct => { - let state = app.state.lock().await; - let existing_peer_ports = local_direct_peer_ports(&app.plan, &state)?; - let existing_peer_identities = local_direct_peer_identities(&app.plan, &state)?; - drop(state); - write_json( - &site_controller_runtime_child_peer_ports_path(&app.plan, child_id), - &existing_peer_ports, - )?; - write_json( - &site_controller_runtime_child_peer_identities_path(&app.plan, child_id), - &existing_peer_identities, - )?; - project_dynamic_child_mesh_scope( - Path::new(&child.artifact_root), - existing_peer_identities - .get(&app.plan.router_identity_id) - .and_then(|identity| identity.mesh_scope.as_deref()), - )?; - if dynamic_route_overlay_path(Path::new(&child.artifact_root)).is_file() { - apply_dynamic_site_router_overlay(&app.plan, &child).await?; - } - let runtime_root = site_controller_runtime_child_runtime_root(&app.plan, child_id); - let storage_root = site_controller_runtime_child_storage_root(&app.plan, child_id); - let child_root = site_controller_runtime_child_root(&app.plan, child_id); - fs::create_dir_all(&runtime_root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", runtime_root.display()))?; - fs::create_dir_all(&storage_root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", storage_root.display()))?; - let process = spawn_detached_child(&child_root, &child_root.join("site.log"), |cmd| { - cmd.arg("run-direct-init") - .arg("--plan") - .arg(Path::new(&child.artifact_root).join("direct-plan.json")) - .arg("--storage-root") - .arg(&storage_root) - .arg("--runtime-root") - .arg(&runtime_root) - .arg("--existing-peer-ports") - .arg(site_controller_runtime_child_peer_ports_path( - &app.plan, child_id, - )) - .arg("--existing-peer-identities") - .arg(site_controller_runtime_child_peer_identities_path( - &app.plan, child_id, - )) - .arg("--skip-router"); - })?; - { - let mut state = app.state.lock().await; - let record = state - .children - .get_mut(&child_id) - .expect("prepared child should remain present"); - record.process_pid = Some(process.id()); - write_json(&app.state_path, &*state)?; - } - wait_for_detached_child_runtime_state( - process.id(), - &direct_runtime_state_path(Path::new(&child.artifact_root)), - site_ready_timeout_for_kind(SiteKind::Direct), - &child_root.join("site.log"), - ) - .await?; - let mut live_children = published_children.clone(); - live_children.push(child.clone()); - let live_peer_ports = overlay_peer_addr_map_from_ports( - &local_direct_peer_ports_for_children(&app.plan, &live_children)?, - ); - let live_peer_identities = - local_direct_peer_identities_for_children(&app.plan, &live_children)?; - write_direct_vm_live_route_overlay_payload( - Path::new(&child.artifact_root), - &child.assigned_components, - &child.proxy_exports, - &child.routed_inputs, - &live_peer_ports, - &live_peer_identities, - )?; - apply_dynamic_site_router_overlay(&app.plan, &child).await?; - let mut state = app.state.lock().await; - { - let record = state - .children - .get_mut(&child_id) - .expect("prepared child should remain present"); - record.process_pid = Some(process.id()); - record.published = true; - } - write_json(&app.state_path, &*state)?; - drop(state); - project_dynamic_direct_router_surface(&app.plan, &child)?; - reconcile_dynamic_site_router_overlays(app).await?; - reconcile_site_proxy_metadata(Path::new(&app.plan.artifact_dir), &site_plan)?; - } - SiteKind::Vm => { - let state = app.state.lock().await; - let existing_peer_ports = local_vm_peer_ports(&app.plan, &state)?; - let existing_peer_identities = local_vm_peer_identities(&app.plan, &state)?; - drop(state); - write_json( - &site_controller_runtime_child_peer_ports_path(&app.plan, child_id), - &existing_peer_ports, - )?; - write_json( - &site_controller_runtime_child_peer_identities_path(&app.plan, child_id), - &existing_peer_identities, - )?; - project_dynamic_child_mesh_scope( - Path::new(&child.artifact_root), - existing_peer_identities - .get(&app.plan.router_identity_id) - .and_then(|identity| identity.mesh_scope.as_deref()), - )?; - if dynamic_route_overlay_path(Path::new(&child.artifact_root)).is_file() { - apply_dynamic_site_router_overlay(&app.plan, &child).await?; - } - let runtime_root = site_controller_runtime_child_runtime_root(&app.plan, child_id); - let storage_root = site_controller_runtime_child_storage_root(&app.plan, child_id); - let child_root = site_controller_runtime_child_root(&app.plan, child_id); - fs::create_dir_all(&runtime_root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", runtime_root.display()))?; - fs::create_dir_all(&storage_root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", storage_root.display()))?; - let process = spawn_detached_child(&child_root, &child_root.join("site.log"), |cmd| { - cmd.arg("run-vm-init") - .arg("--plan") - .arg(Path::new(&child.artifact_root).join("vm-plan.json")) - .arg("--storage-root") - .arg(&storage_root) - .arg("--runtime-root") - .arg(&runtime_root) - .arg("--existing-peer-ports") - .arg(site_controller_runtime_child_peer_ports_path( - &app.plan, child_id, - )) - .arg("--existing-peer-identities") - .arg(site_controller_runtime_child_peer_identities_path( - &app.plan, child_id, - )) - .arg("--skip-router"); - })?; - { - let mut state = app.state.lock().await; - let record = state - .children - .get_mut(&child_id) - .expect("prepared child should remain present"); - record.process_pid = Some(process.id()); - write_json(&app.state_path, &*state)?; - } - wait_for_detached_child_runtime_state( - process.id(), - &Path::new(&child.artifact_root) - .join(".amber") - .join("vm-runtime.json"), - vm_endpoint_forward_ready_timeout(), - &child_root.join("site.log"), - ) - .await?; - wait_for_detached_vm_child_endpoints_ready( - process.id(), - Path::new(&child.artifact_root), - &runtime_root, - vm_endpoint_forward_ready_timeout(), - &child_root.join("site.log"), - )?; - let mut live_children = published_children.clone(); - live_children.push(child.clone()); - let live_peer_ports = overlay_peer_addr_map_from_ports( - &local_vm_peer_ports_for_children(&app.plan, &live_children)?, - ); - let live_peer_identities = - local_vm_peer_identities_for_children(&app.plan, &live_children)?; - write_direct_vm_live_route_overlay_payload( - Path::new(&child.artifact_root), - &child.assigned_components, - &child.proxy_exports, - &child.routed_inputs, - &live_peer_ports, - &live_peer_identities, - )?; - apply_dynamic_site_router_overlay(&app.plan, &child).await?; - let mut state = app.state.lock().await; - { - let record = state - .children - .get_mut(&child_id) - .expect("prepared child should remain present"); - record.process_pid = Some(process.id()); - record.published = true; - } - write_json(&app.state_path, &*state)?; - drop(state); - project_dynamic_vm_router_surface(&app.plan, &child)?; - reconcile_dynamic_site_router_overlays(app).await?; - reconcile_site_proxy_metadata(Path::new(&app.plan.artifact_dir), &site_plan)?; - } - SiteKind::Compose => { - let mut issuer_children = published_children.clone(); - issuer_children.push(child.clone()); - reconcile_dynamic_site_router_overlays_for_children( - app, - &published_children, - &issuer_children, - ) - .await?; - apply_dynamic_site_router_overlay(&app.plan, &child).await?; - let metadata = load_dynamic_compose_child_metadata(Path::new(&child.artifact_root))?; - let compose_project = app.plan.compose_project.as_deref().ok_or_else(|| { - miette::miette!( - "compose site `{}` is missing its compose project name", - app.plan.site_id - ) - })?; - let (sidecar_services, workload_services): (Vec<_>, Vec<_>) = metadata - .services - .iter() - .cloned() - .partition(|service| is_compose_component_sidecar_service(service)); - if !sidecar_services.is_empty() { - let status = - compose_command(Some(compose_project), Path::new(&child.artifact_root)) - .envs(app.plan.launch_env.clone()) - .arg("up") - .arg("-d") - .args(&sidecar_services) - .status() - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to publish compose child sidecars on site `{}`", - app.plan.site_id - ) - })?; - if !status.success() { - return Err(miette::miette!( - "compose child sidecar publish on site `{}` failed with status {status}", - app.plan.site_id - )); - } - wait_for_compose_services_running( - compose_project, - Path::new(&child.artifact_root), - &sidecar_services, - site_ready_timeout_for_kind(SiteKind::Compose), - ) - .await?; - } - if !workload_services.is_empty() { - let status = - compose_command(Some(compose_project), Path::new(&child.artifact_root)) - .envs(app.plan.launch_env.clone()) - .arg("up") - .arg("-d") - .args(&workload_services) - .status() - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to publish compose child workloads on site `{}`", - app.plan.site_id - ) - })?; - if !status.success() { - return Err(miette::miette!( - "compose child workload publish on site `{}` failed with status {status}", - app.plan.site_id - )); - } - } - wait_for_compose_services_running( - compose_project, - Path::new(&child.artifact_root), - &metadata.readiness_services, - site_ready_timeout_for_kind(SiteKind::Compose), - ) - .await?; - let mut state = app.state.lock().await; - let record = state - .children - .get_mut(&child_id) - .expect("prepared child should remain present"); - record.published = true; - write_json(&app.state_path, &*state)?; - drop(state); - reconcile_dynamic_site_router_overlays(app).await?; - } - SiteKind::Kubernetes => { - let artifact_root = Path::new(&child.artifact_root); - let supervisor_plan = prepare_kubernetes_artifact_for_apply(&app.plan, artifact_root)?; - ensure_kubernetes_namespace(&supervisor_plan)?; - let status = kubectl_command(app.plan.context.as_deref()) - .current_dir(artifact_root) - .arg("apply") - .arg("-k") - .arg(".") - .status() - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to publish kubernetes site `{}`", app.plan.site_id) - })?; - if !status.success() { - return Err(miette::miette!( - "kubernetes site `{}` publish failed with status {status}", - app.plan.site_id - )); - } - ensure_kubernetes_workloads_ready(&supervisor_plan)?; - let mut live_children = published_children.clone(); - live_children.push(child.clone()); - let live_peer_identities = local_kubernetes_peer_identities(&app.plan, &live_children)?; - write_kubernetes_live_route_overlay_payload( - artifact_root, - &child.assigned_components, - &child.proxy_exports, - &child.routed_inputs, - &live_peer_identities, - )?; - apply_dynamic_site_router_overlay(&app.plan, &child).await?; - let mut state = app.state.lock().await; - let record = state - .children - .get_mut(&child_id) - .expect("prepared child should remain present"); - record.published = true; - write_json(&app.state_path, &*state)?; - drop(state); - reconcile_dynamic_site_router_overlays(app).await?; - wait_for_kubernetes_site_router_ready( - &app.plan, - site_ready_timeout_for_kind(SiteKind::Kubernetes), - ) - .await?; - } - } - - Ok(()) -} - -async fn site_controller_runtime_rollback_child( - app: &SiteControllerRuntimeApp, - child_id: u64, -) -> Result<()> { - let child = { - let mut state = app.state.lock().await; - let removed = state.children.remove(&child_id); - write_json(&app.state_path, &*state)?; - removed - }; - let Some(child) = child else { - return Ok(()); - }; - if let Some(pid) = child.process_pid { - terminate_pid(pid, site_ready_timeout_for_kind(app.plan.kind))?; - } - remove_dir_if_exists(&site_controller_runtime_child_root(&app.plan, child_id)) -} - -async fn site_controller_runtime_destroy_child( - app: &SiteControllerRuntimeApp, - child_id: u64, - desired_site_plan: Option, -) -> Result<()> { - let child = { - let state = app.state.lock().await; - state.children.get(&child_id).cloned() - }; - if let Some(child) = child.as_ref() - && matches!( - app.plan.kind, - SiteKind::Direct | SiteKind::Vm | SiteKind::Compose | SiteKind::Kubernetes - ) - { - revoke_dynamic_site_router_overlay(&app.plan, child).await?; - } - if let Some(pid) = child.as_ref().and_then(|child| child.process_pid) { - terminate_pid(pid, site_ready_timeout_for_kind(app.plan.kind))?; - } - - match app.plan.kind { - SiteKind::Compose => { - let _ = desired_site_plan; - let child = child.as_ref().ok_or_else(|| { - miette::miette!("site controller runtime child {child_id} is not prepared") - })?; - let metadata = load_dynamic_compose_child_metadata(Path::new(&child.artifact_root))?; - let compose_project = app.plan.compose_project.as_deref().ok_or_else(|| { - miette::miette!( - "compose site `{}` is missing its compose project name", - app.plan.site_id - ) - })?; - if !metadata.services.is_empty() { - let status = - compose_command(Some(compose_project), Path::new(&child.artifact_root)) - .envs(app.plan.launch_env.clone()) - .arg("rm") - .arg("--stop") - .arg("--force") - .arg("-v") - .args(&metadata.services) - .status() - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to destroy compose child on site `{}`", - app.plan.site_id - ) - })?; - if !status.success() { - return Err(miette::miette!( - "compose child destroy on site `{}` failed with status {status}", - app.plan.site_id - )); - } - } - } - SiteKind::Kubernetes => { - let _ = desired_site_plan; - let child = child.as_ref().ok_or_else(|| { - miette::miette!("site controller runtime child {child_id} is not prepared") - })?; - let artifact_root = Path::new(&child.artifact_root); - let files = read_artifact_snapshot(artifact_root)?; - let destroy_bundle = project_kubernetes_dynamic_child_destroy_artifact_files(&files)?; - replace_artifact_snapshot(artifact_root, &destroy_bundle)?; - let supervisor_plan = prepare_kubernetes_artifact_for_apply(&app.plan, artifact_root)?; - let workloads = supervisor::kubernetes_expected_workloads(artifact_root)?; - let status = kubectl_command(app.plan.context.as_deref()) - .current_dir(artifact_root) - .arg("delete") - .arg("-k") - .arg(".") - .arg("--ignore-not-found=true") - .status() - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to destroy kubernetes child on site `{}`", - app.plan.site_id - ) - })?; - if !status.success() { - return Err(miette::miette!( - "kubernetes child destroy on site `{}` failed with status {status}", - app.plan.site_id - )); - } - wait_for_kubernetes_artifact_workloads_deleted( - app.plan.context.as_deref(), - supervisor_plan - .kubernetes_namespace - .as_deref() - .expect("kubernetes supervisor plan should include a namespace"), - &workloads, - &app.plan.site_id, - )?; - wait_for_kubernetes_site_router_ready( - &app.plan, - site_ready_timeout_for_kind(SiteKind::Kubernetes), - ) - .await?; - } - SiteKind::Direct | SiteKind::Vm => { - if let Some(desired) = desired_site_plan.as_ref() { - reconcile_site_proxy_metadata(Path::new(&app.plan.artifact_dir), desired)?; - } - } - } - - let mut state = app.state.lock().await; - state.children.remove(&child_id); - write_json(&app.state_path, &*state)?; - drop(state); - if matches!( - app.plan.kind, - SiteKind::Direct | SiteKind::Vm | SiteKind::Compose | SiteKind::Kubernetes - ) { - reconcile_dynamic_site_router_overlays(app).await?; - } - remove_dir_if_exists(&site_controller_runtime_child_root(&app.plan, child_id)) -} - -fn replace_artifact_snapshot(root: &Path, files: &BTreeMap) -> Result<()> { - fs::create_dir_all(root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", root.display()))?; - let expected = files.keys().map(PathBuf::from).collect::>(); - for path in walk_files(root)? { - let relative = path - .strip_prefix(root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to relativize {}", path.display()))? - .to_path_buf(); - if !expected.contains(&relative) { - fs::remove_file(&path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to remove stale artifact {}", path.display()))?; - } - } - for (relative, contents) in files { - let path = root.join(relative); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - fs::write(&path, contents) - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", path.display()))?; - } - Ok(()) -} - -fn wait_for_named_kubernetes_resource_deleted( - context: Option<&str>, - namespace: &str, - resource_kind: &str, - name: &str, - label: &str, -) -> Result<()> { - let resource = format!("{resource_kind}/{name}"); - let timeout = format!("{}s", KUBERNETES_WORKLOAD_READY_TIMEOUT.as_secs().max(1)); - let output = kubectl_command(context) - .args([ - "-n", - namespace, - "wait", - "--for=delete", - "--timeout", - timeout.as_str(), - resource.as_str(), - ]) - .output() - .into_diagnostic() - .wrap_err_with(|| format!("{label}: wait for {resource} deletion"))?; - if output.status.success() { - return Ok(()); - } - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - if stderr.contains("not found") { - return Ok(()); - } - let detail = if stderr.is_empty() { - format!("status {}", output.status) - } else { - stderr - }; - Err(miette::miette!("{label} failed: {detail}")) -} - -fn wait_for_kubernetes_artifact_workloads_deleted( - context: Option<&str>, - namespace: &str, - workloads: &supervisor::KubernetesArtifactWorkloads, - site_id: &str, -) -> Result<()> { - let label = format!("wait for kubernetes child workload deletion on site `{site_id}`"); - for job in &workloads.jobs { - wait_for_named_kubernetes_resource_deleted(context, namespace, "job", job, &label)?; - } - for deployment in &workloads.deployments { - wait_for_named_kubernetes_resource_deleted( - context, - namespace, - "deployment", - deployment, - &label, - )?; - } - Ok(()) -} - -async fn wait_for_kubernetes_site_router_ready( - plan: &SiteControllerRuntimePlan, - timeout: Duration, -) -> Result<()> { - debug_assert_eq!(plan.kind, SiteKind::Kubernetes); - - let state_path = Path::new(&plan.site_state_root).join("manager-state.json"); - let deadline = Instant::now() + timeout; - loop { - if state_path.is_file() { - let state: SiteManagerState = read_json(&state_path, "site manager state")?; - if matches!(state.status, SiteLifecycleStatus::Failed) { - return Err(miette::miette!( - "kubernetes site `{}` failed while waiting for router recovery: {}", - plan.site_id, - state - .last_error - .unwrap_or_else(|| "unknown failure".to_string()) - )); - } - if matches!(state.status, SiteLifecycleStatus::Running) - && let (Some(control), Some(mesh_addr)) = ( - state.router_control.as_deref(), - state.router_mesh_addr.as_deref(), - ) - { - let control_addr: SocketAddr = - control.parse().into_diagnostic().wrap_err_with(|| { - format!("invalid kubernetes router control addr `{control}`") - })?; - let mesh_addr: SocketAddr = - mesh_addr.parse().into_diagnostic().wrap_err_with(|| { - format!("invalid kubernetes router mesh addr `{mesh_addr}`") - })?; - if probe_kubernetes_router_identity(control_addr, Duration::from_millis(250)) - .await? - && router_mesh_listener_ready(mesh_addr).await - { - return Ok(()); - } - } - } - if Instant::now() >= deadline { - return Err(miette::miette!( - "timed out waiting for kubernetes site `{}` router recovery", - plan.site_id - )); - } - sleep(Duration::from_millis(200)).await; - } -} - -async fn probe_kubernetes_router_identity(addr: SocketAddr, timeout: Duration) -> Result { - let mut stream = match tokio::time::timeout(timeout, tokio::net::TcpStream::connect(addr)).await - { - Ok(Ok(stream)) => stream, - Ok(Err(err)) - if matches!( - err.kind(), - std::io::ErrorKind::ConnectionRefused - | std::io::ErrorKind::ConnectionReset - | std::io::ErrorKind::ConnectionAborted - | std::io::ErrorKind::TimedOut - | std::io::ErrorKind::UnexpectedEof - | std::io::ErrorKind::NotConnected - ) => - { - return Ok(false); - } - Ok(Err(err)) => { - return Err(miette::miette!( - "failed to connect to kubernetes router control at {addr}: {err}" - )); - } - Err(_) => return Ok(false), - }; - - let request = b"GET /identity HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"; - match tokio::time::timeout(timeout, stream.write_all(request)).await { - Ok(Ok(())) => {} - Ok(Err(err)) - if matches!( - err.kind(), - std::io::ErrorKind::BrokenPipe - | std::io::ErrorKind::ConnectionReset - | std::io::ErrorKind::ConnectionAborted - | std::io::ErrorKind::TimedOut - | std::io::ErrorKind::UnexpectedEof - | std::io::ErrorKind::NotConnected - ) => - { - return Ok(false); - } - Ok(Err(err)) => { - return Err(miette::miette!( - "failed to write kubernetes router identity probe to {addr}: {err}" - )); - } - Err(_) => return Ok(false), - } - - let deadline = Instant::now() + timeout; - let mut buf = Vec::new(); - loop { - if let Some(end) = find_header_end(&buf) - && let Some(content_length) = router_identity_probe_content_length(&buf[..end])? - { - let body_len = buf.len().saturating_sub(end + 4); - if body_len >= content_length { - return parse_router_identity_probe_response(&buf, end); - } - } - let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - return Ok(false); - }; - let mut chunk = [0u8; 1024]; - match tokio::time::timeout(remaining, stream.read(&mut chunk)).await { - Ok(Ok(0)) => return Ok(false), - Ok(Ok(read)) => buf.extend_from_slice(&chunk[..read]), - Ok(Err(err)) - if matches!( - err.kind(), - std::io::ErrorKind::ConnectionReset - | std::io::ErrorKind::ConnectionAborted - | std::io::ErrorKind::TimedOut - | std::io::ErrorKind::UnexpectedEof - | std::io::ErrorKind::NotConnected - ) => - { - return Ok(false); - } - Ok(Err(err)) => { - return Err(miette::miette!( - "failed to read kubernetes router identity probe from {addr}: {err}" - )); - } - Err(_) => return Ok(false), - } - } -} - -fn router_identity_probe_content_length(header: &[u8]) -> Result> { - let header = std::str::from_utf8(header) - .into_diagnostic() - .wrap_err("router identity probe returned a non-UTF-8 HTTP header")?; - Ok(header.lines().find_map(|line| { - let (name, value) = line.split_once(':')?; - name.trim() - .eq_ignore_ascii_case("content-length") - .then_some(value.trim()) - .and_then(|value| value.parse::().ok()) - })) -} - -fn parse_router_identity_probe_response(buf: &[u8], header_end: usize) -> Result { - let header = std::str::from_utf8(&buf[..header_end]) - .into_diagnostic() - .wrap_err("router identity probe returned a non-UTF-8 HTTP header")?; - let status = header - .lines() - .next() - .and_then(|line| line.split_whitespace().nth(1)) - .and_then(|code| code.parse::().ok()); - if status != Some(200) { - return Ok(false); - } - let body = std::str::from_utf8(&buf[header_end + 4..]) - .into_diagnostic() - .wrap_err("router identity probe returned a non-UTF-8 body")?; - Ok(serde_json::from_str::(body.trim()).is_ok()) -} - -fn filter_direct_stage_plan(artifact_root: &Path, component_ids: &[usize]) -> Result<()> { - let keep = component_ids.iter().copied().collect::>(); - let plan_path = artifact_root.join("direct-plan.json"); - let mut plan: DirectPlan = read_json(&plan_path, "direct plan")?; - plan.components - .retain(|component| keep.contains(&component.id)); - plan.startup_order - .retain(|component_id| keep.contains(component_id)); - filter_dynamic_runtime_addresses(&mut plan.runtime_addresses, &keep); - let keep_mesh_output_dirs = plan - .components - .iter() - .filter_map(|component| { - Path::new(&component.sidecar.mesh_config_path) - .parent() - .map(path_to_forward_slash_string) - }) - .collect::>(); - plan.router = None; - write_json(&plan_path, &plan)?; - filter_dynamic_mesh_provision_plan(artifact_root, &keep_mesh_output_dirs) -} - -fn filter_vm_stage_plan(artifact_root: &Path, component_ids: &[usize]) -> Result<()> { - let keep = component_ids.iter().copied().collect::>(); - let plan_path = artifact_root.join("vm-plan.json"); - let mut plan: VmPlan = read_json(&plan_path, "vm plan")?; - plan.components - .retain(|component| keep.contains(&component.id)); - plan.startup_order - .retain(|component_id| keep.contains(component_id)); - filter_dynamic_runtime_addresses(&mut plan.runtime_addresses, &keep); - let keep_mesh_output_dirs = plan - .components - .iter() - .filter_map(|component| { - Path::new(&component.mesh_config_path) - .parent() - .map(path_to_forward_slash_string) - }) - .collect::>(); - plan.router = None; - write_json(&plan_path, &plan)?; - filter_dynamic_mesh_provision_plan(artifact_root, &keep_mesh_output_dirs) -} - -fn filter_dynamic_runtime_addresses( - runtime_addresses: &mut amber_compiler::reporter::direct::DirectRuntimeAddressPlan, - keep: &BTreeSet, -) { - runtime_addresses - .slots_by_scope - .retain(|scope, _| keep.contains(scope)); - for sources in runtime_addresses.slots_by_scope.values_mut() { - sources.retain(|_, source| keep.contains(&dynamic_runtime_source_component_id(source))); - } - runtime_addresses - .slot_items_by_scope - .retain(|scope, _| keep.contains(scope)); - for sources in runtime_addresses.slot_items_by_scope.values_mut() { - sources.retain(|_, items| { - items.retain(|source| keep.contains(&dynamic_runtime_source_component_id(source))); - !items.is_empty() - }); - } -} - -fn dynamic_runtime_source_component_id(source: &DirectRuntimeUrlSource) -> usize { - match source { - DirectRuntimeUrlSource::Slot { component_id, .. } - | DirectRuntimeUrlSource::SlotItem { component_id, .. } => *component_id, - } -} - -fn path_to_forward_slash_string(path: &Path) -> String { - path.components() - .fold(String::new(), |mut rendered, component| { - if !rendered.is_empty() { - rendered.push('/'); - } - rendered.push_str(&component.as_os_str().to_string_lossy()); - rendered - }) -} - -fn filter_dynamic_mesh_provision_plan( - artifact_root: &Path, - keep_component_output_dirs: &BTreeSet, -) -> Result<()> { - let plan_path = artifact_root.join("mesh-provision-plan.json"); - let mut plan: MeshProvisionPlan = read_json(&plan_path, "mesh provision plan")?; - let mut kept_component_ids = BTreeSet::new(); - let mut router_target = None; - let mut filtered_targets = Vec::with_capacity(plan.targets.len()); - for target in plan.targets { - match target.kind { - MeshProvisionTargetKind::Component => { - let MeshProvisionOutput::Filesystem { dir } = &target.output else { - return Err(miette::miette!( - "dynamic mesh target {} does not use filesystem output", - target.config.identity.id - )); - }; - if keep_component_output_dirs.contains(dir) { - kept_component_ids.insert(target.config.identity.id.clone()); - filtered_targets.push(target); - } - } - MeshProvisionTargetKind::Router => { - router_target = Some(target); - } - } - } - - let Some(mut router_target) = router_target else { - return Err(miette::miette!( - "dynamic mesh provision plan {} is missing a router target", - plan_path.display() - )); - }; - filter_dynamic_router_target(&mut router_target, &kept_component_ids); - filtered_targets.push(router_target); - plan.targets = filtered_targets; - write_json(&plan_path, &plan) -} - -async fn wait_for_detached_child_runtime_state( - pid: u32, - state_path: &Path, - timeout: Duration, - log_path: &Path, -) -> Result<()> { - let deadline = Instant::now() + timeout; - while Instant::now() < deadline { - if state_path.is_file() { - return Ok(()); - } - if !pid_is_alive(pid) { - let log = fs::read_to_string(log_path).unwrap_or_default(); - return Err(miette::miette!( - "dynamic child runtime exited before becoming ready\nlog ({}):\n{}", - log_path.display(), - log - )); - } - sleep(Duration::from_millis(100)).await; - } - let log = fs::read_to_string(log_path).unwrap_or_default(); - Err(miette::miette!( - "timed out waiting for dynamic child runtime state {}\nlog ({}):\n{}", - state_path.display(), - log_path.display(), - log - )) -} - -fn wait_for_detached_vm_child_endpoints_ready( - pid: u32, - artifact_root: &Path, - runtime_root: &Path, - timeout: Duration, - log_path: &Path, -) -> Result<()> { - let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; - let deadline = Instant::now() + timeout; - for component in &plan.components { - let config: MeshConfigPublic = read_json( - &runtime_root.join(&component.mesh_config_path), - "mesh config", - )?; - for route in config.inbound { - let InboundTarget::Local { port: host_port } = route.target else { - continue; - }; - let addr = SocketAddr::from(([127, 0, 0, 1], host_port)); - loop { - let ready = match route.protocol { - MeshProtocol::Http => endpoint_returns_http_response( - addr, - Duration::from_millis(250), - Duration::from_millis(250), - ), - MeshProtocol::Tcp => endpoint_accepts_stable_connection( - addr, - Duration::from_millis(250), - Duration::from_millis(250), - ), - }; - if ready { - break; - } - if !pid_is_alive(pid) { - let log = fs::read_to_string(log_path).unwrap_or_default(); - return Err(miette::miette!( - "dynamic vm child runtime exited before component {} endpoint {} became \ - ready\nlog ({}):\n{}", - component.moniker, - addr, - log_path.display(), - log - )); - } - if Instant::now() >= deadline { - let log = fs::read_to_string(log_path).unwrap_or_default(); - let protocol = match route.protocol { - MeshProtocol::Http => "http", - MeshProtocol::Tcp => "tcp", - }; - return Err(miette::miette!( - "timed out waiting for dynamic vm child component {} {} endpoint {} to \ - become ready\nlog ({}):\n{}", - component.moniker, - protocol, - addr, - log_path.display(), - log - )); - } - std::thread::sleep(Duration::from_millis(50)); - } - } - } - Ok(()) -} - -fn local_direct_peer_ports( - plan: &SiteControllerRuntimePlan, - state: &SiteControllerRuntimeState, -) -> Result> { - let children = state - .children - .values() - .filter(|child| child.published) - .cloned() - .collect::>(); - local_direct_peer_ports_for_children(plan, &children) -} - -fn local_direct_peer_ports_for_children( - plan: &SiteControllerRuntimePlan, - children: &[SiteControllerRuntimeChildRecord], -) -> Result> { - let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { - miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) - })?); - let mut peers = BTreeMap::new(); - peers.insert( - plan.router_identity_id.clone(), - plan.router_mesh_port.ok_or_else(|| { - miette::miette!( - "direct site `{}` is missing its router mesh port", - plan.site_id - ) - })?, - ); - peers.extend(direct_peer_ports_for_artifact( - Path::new(&plan.artifact_dir), - site_runtime_root, - )?); - for child in children { - peers.extend(direct_peer_ports_for_artifact( - Path::new(&child.artifact_root), - &site_controller_runtime_child_runtime_root(plan, child.child_id), - )?); - } - Ok(peers) -} - -fn local_direct_peer_identities( - plan: &SiteControllerRuntimePlan, - state: &SiteControllerRuntimeState, -) -> Result> { - let children = state - .children - .values() - .filter(|child| child.published) - .cloned() - .collect::>(); - local_direct_peer_identities_for_children(plan, &children) -} - -fn local_direct_peer_identities_for_children( - plan: &SiteControllerRuntimePlan, - children: &[SiteControllerRuntimeChildRecord], -) -> Result> { - let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { - miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) - })?); - let mut peers = BTreeMap::new(); - let router = - direct_router_identity_for_artifact(Path::new(&plan.artifact_dir), site_runtime_root)? - .ok_or_else(|| { - miette::miette!( - "direct site `{}` is missing its live router identity", - plan.site_id - ) - })?; - peers.insert(router.id.clone(), router); - peers.extend(direct_peer_identities_for_artifact( - Path::new(&plan.artifact_dir), - site_runtime_root, - )?); - for child in children { - peers.extend(direct_peer_identities_for_artifact( - Path::new(&child.artifact_root), - &site_controller_runtime_child_runtime_root(plan, child.child_id), - )?); - } - Ok(peers) -} - -fn direct_peer_ports_for_artifact( - artifact_root: &Path, - runtime_root: &Path, -) -> Result> { - let plan: DirectPlan = read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; - let mut peers = BTreeMap::new(); - for component in &plan.components { - let config: MeshConfigPublic = read_json( - &runtime_root.join(&component.sidecar.mesh_config_path), - "mesh config", - )?; - peers.insert(config.identity.id, config.mesh_listen.port()); - } - Ok(peers) -} - -fn direct_peer_identities_for_artifact( - artifact_root: &Path, - runtime_root: &Path, -) -> Result> { - let plan: DirectPlan = read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; - let mut peers = BTreeMap::new(); - for component in &plan.components { - let config: MeshConfigPublic = read_json( - &runtime_root.join(&component.sidecar.mesh_config_path), - "mesh config", - )?; - peers.insert(config.identity.id.clone(), config.identity); - } - Ok(peers) -} - -fn direct_router_identity_for_artifact( - artifact_root: &Path, - runtime_root: &Path, -) -> Result> { - let plan: DirectPlan = read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; - let Some(router) = plan.router.as_ref() else { - return Ok(None); - }; - let config: MeshConfigPublic = - read_json(&runtime_root.join(&router.mesh_config_path), "mesh config")?; - Ok(Some(config.identity)) -} - -fn local_vm_peer_ports( - plan: &SiteControllerRuntimePlan, - state: &SiteControllerRuntimeState, -) -> Result> { - let children = state - .children - .values() - .filter(|child| child.published) - .cloned() - .collect::>(); - local_vm_peer_ports_for_children(plan, &children) -} - -fn local_vm_peer_ports_for_children( - plan: &SiteControllerRuntimePlan, - children: &[SiteControllerRuntimeChildRecord], -) -> Result> { - let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { - miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) - })?); - let mut peers = BTreeMap::new(); - peers.insert( - plan.router_identity_id.clone(), - plan.router_mesh_port.ok_or_else(|| { - miette::miette!("vm site `{}` is missing its router mesh port", plan.site_id) - })?, - ); - peers.extend(vm_peer_ports_for_artifact( - Path::new(&plan.artifact_dir), - site_runtime_root, - )?); - for child in children { - peers.extend(vm_peer_ports_for_artifact( - Path::new(&child.artifact_root), - &site_controller_runtime_child_runtime_root(plan, child.child_id), - )?); - } - Ok(peers) -} - -fn local_vm_peer_identities( - plan: &SiteControllerRuntimePlan, - state: &SiteControllerRuntimeState, -) -> Result> { - let children = state - .children - .values() - .filter(|child| child.published) - .cloned() - .collect::>(); - local_vm_peer_identities_for_children(plan, &children) -} - -fn local_vm_peer_identities_for_children( - plan: &SiteControllerRuntimePlan, - children: &[SiteControllerRuntimeChildRecord], -) -> Result> { - let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { - miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) - })?); - let mut peers = BTreeMap::new(); - let router = vm_router_identity_for_artifact(Path::new(&plan.artifact_dir), site_runtime_root)? - .ok_or_else(|| { - miette::miette!( - "vm site `{}` is missing its live router identity", - plan.site_id - ) - })?; - peers.insert(router.id.clone(), router); - peers.extend(vm_peer_identities_for_artifact( - Path::new(&plan.artifact_dir), - site_runtime_root, - )?); - for child in children { - peers.extend(vm_peer_identities_for_artifact( - Path::new(&child.artifact_root), - &site_controller_runtime_child_runtime_root(plan, child.child_id), - )?); - } - Ok(peers) -} - -fn vm_peer_ports_for_artifact( - artifact_root: &Path, - runtime_root: &Path, -) -> Result> { - let state = load_vm_runtime_state_for_artifact(artifact_root, runtime_root)?; - let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; - let mut peers = BTreeMap::new(); - for component in &plan.components { - let config: MeshConfigPublic = read_json( - &runtime_root.join(&component.mesh_config_path), - "mesh config", - )?; - let port = state - .component_mesh_port_by_id - .get(&component.id) - .copied() - .ok_or_else(|| { - miette::miette!( - "vm runtime state is missing mesh port for component {}", - component.moniker - ) - })?; - peers.insert(config.identity.id, port); - } - Ok(peers) -} - -fn load_vm_runtime_state_for_artifact( - artifact_root: &Path, - runtime_root: &Path, -) -> Result { - let state_path = artifact_root.join(".amber").join("vm-runtime.json"); - if state_path.is_file() { - return read_json(&state_path, "vm runtime state"); - } - - let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; - let mut state = VmRuntimeState::default(); - for component in &plan.components { - let config: MeshConfigPublic = read_json( - &runtime_root.join(&component.mesh_config_path), - "mesh config", - )?; - state - .component_mesh_port_by_id - .insert(component.id, config.mesh_listen.port()); - } - if let Some(router) = &plan.router { - let config: MeshConfigPublic = - read_json(&runtime_root.join(&router.mesh_config_path), "mesh config")?; - state.router_mesh_port = Some(config.mesh_listen.port()); - } - write_vm_runtime_state(artifact_root, &state)?; - Ok(state) -} - -fn vm_peer_identities_for_artifact( - artifact_root: &Path, - runtime_root: &Path, -) -> Result> { - let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; - let mut peers = BTreeMap::new(); - for component in &plan.components { - let config: MeshConfigPublic = read_json( - &runtime_root.join(&component.mesh_config_path), - "mesh config", - )?; - peers.insert(config.identity.id.clone(), config.identity); - } - Ok(peers) -} - -fn vm_router_identity_for_artifact( - artifact_root: &Path, - runtime_root: &Path, -) -> Result> { - let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; - let Some(router) = plan.router.as_ref() else { - return Ok(None); - }; - let config: MeshConfigPublic = - read_json(&runtime_root.join(&router.mesh_config_path), "mesh config")?; - Ok(Some(config.identity)) -} - -fn terminate_pid(pid: u32, timeout: Duration) -> Result<()> { - #[cfg(unix)] - { - terminate_detached_runtime(pid, timeout) - } - - #[cfg(not(unix))] - { - send_sigterm(pid); - let deadline = Instant::now() + timeout; - while Instant::now() < deadline { - if !pid_is_alive(pid) { - return Ok(()); - } - std::thread::sleep(Duration::from_millis(100)); - } - Ok(()) - } -} - -#[cfg(unix)] -fn terminate_detached_runtime(root_pid: u32, timeout: Duration) -> Result<()> { - let mut tracked = process_tree_postorder(root_pid)? - .into_iter() - .filter(|pid| pid_is_alive(*pid)) - .collect::>(); - if tracked.is_empty() { - return Ok(()); - } - - send_signal_to_process_group(root_pid, libc::SIGTERM); - send_signal_to_pids(&tracked, libc::SIGTERM); - - let deadline = Instant::now() + timeout; - while Instant::now() < deadline { - tracked.retain(|pid| pid_is_alive(*pid)); - if tracked.is_empty() { - return Ok(()); - } - std::thread::sleep(Duration::from_millis(100)); - } - - send_signal_to_process_group(root_pid, libc::SIGKILL); - send_signal_to_pids(&tracked, libc::SIGKILL); - - let force_deadline = Instant::now() + Duration::from_secs(2); - while Instant::now() < force_deadline { - tracked.retain(|pid| pid_is_alive(*pid)); - if tracked.is_empty() { - return Ok(()); - } - std::thread::sleep(Duration::from_millis(100)); - } - - Err(miette::miette!( - "failed to terminate detached runtime rooted at pid {root_pid}; surviving processes: {}", - tracked - .iter() - .map(u32::to_string) - .collect::>() - .join(", ") - )) -} - -fn remove_dir_if_exists(path: &Path) -> Result<()> { - if !path.exists() { - return Ok(()); - } - fs::remove_dir_all(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to remove {}", path.display())) -} - -fn prepare_kubernetes_artifact_namespace( - run_id: &str, - site_id: &str, - artifact_dir: &Path, -) -> Result { - let namespace = kubernetes_namespace_name(run_id, site_id); - let kustomization = artifact_dir.join("kustomization.yaml"); - let contents = fs::read_to_string(&kustomization) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", kustomization.display()))?; - - let mut saw_namespace = false; - let mut out = String::new(); - for line in contents.lines() { - if line.trim_start().starts_with("namespace:") { - saw_namespace = true; - out.push_str("namespace: "); - out.push_str(&namespace); - } else { - out.push_str(line); - } - out.push('\n'); - } - if !saw_namespace { - if !out.is_empty() && !out.ends_with('\n') { - out.push('\n'); - } - out.push_str("namespace: "); - out.push_str(&namespace); - out.push('\n'); - } - if out != contents { - fs::write(&kustomization, out) - .into_diagnostic() - .wrap_err_with(|| format!("failed to update {}", kustomization.display()))?; - } - - Ok(namespace) -} - -fn site_supervisor_plan_for_actuator( - plan: &SiteControllerRuntimePlan, - artifact_dir: &Path, - kubernetes_namespace: Option, -) -> SiteSupervisorPlan { - SiteSupervisorPlan { - schema: SITE_PLAN_SCHEMA.to_string(), - version: SITE_PLAN_VERSION, - run_id: plan.run_id.clone(), - mesh_scope: plan.mesh_scope.clone(), - run_root: plan.run_root.clone(), - coordinator_pid: 0, - site_id: plan.site_id.clone(), - kind: plan.kind, - artifact_dir: artifact_dir.display().to_string(), - site_state_root: plan.site_state_root.clone(), - storage_root: plan.storage_root.clone(), - runtime_root: plan.runtime_root.clone(), - router_mesh_port: plan.router_mesh_port, - compose_project: plan.compose_project.clone(), - kubernetes_namespace, - context: plan.context.clone(), - port_forward_mesh_port: None, - port_forward_control_port: None, - observability_endpoint: plan.observability_endpoint.clone(), - site_controller_plan_path: None, - site_controller_url: None, - launch_env: plan.launch_env.clone(), - } -} - -fn prepare_kubernetes_artifact_for_apply( - plan: &SiteControllerRuntimePlan, - artifact_dir: &Path, -) -> Result { - debug_assert_eq!(plan.kind, SiteKind::Kubernetes); - let namespace = - prepare_kubernetes_artifact_namespace(&plan.run_id, &plan.site_id, artifact_dir)?; - patch_site_artifacts( - artifact_dir, - &plan.run_id, - &plan.site_id, - plan.kind, - &plan.launch_env, - plan.observability_endpoint.as_deref(), - )?; - Ok(site_supervisor_plan_for_actuator( - plan, - artifact_dir, - Some(namespace), - )) -} - -fn kubernetes_namespace_name(run_id: &str, site_id: &str) -> String { - let raw = format!("amber-{run_id}-{site_id}"); - let mut out = String::with_capacity(raw.len().min(63)); - let mut last_was_dash = false; - - for ch in raw.chars() { - let next = if ch.is_ascii_alphanumeric() { - ch.to_ascii_lowercase() - } else { - '-' - }; - if next == '-' { - if out.is_empty() || last_was_dash { - continue; - } - last_was_dash = true; - } else { - last_was_dash = false; - } - out.push(next); - if out.len() == 63 { - break; - } - } - while out.ends_with('-') { - out.pop(); - } - if out.is_empty() { - "amber".to_string() - } else { - out - } -} - -async fn wait_for_compose_services_running( - compose_project: &str, - artifact_root: &Path, - services: &[String], - timeout: Duration, -) -> Result<()> { - if services.is_empty() { - return Ok(()); - } - let deadline = Instant::now() + timeout; - loop { - let output = compose_command(Some(compose_project), artifact_root) - .arg("ps") - .arg("--services") - .arg("--status") - .arg("running") - .args(services) - .output() - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to query compose child services in {}", - artifact_root.display() - ) - })?; - if output.status.success() { - let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); - let running = stdout - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()) - .collect::>(); - if services - .iter() - .all(|service| running.contains(service.as_str())) - { - return Ok(()); - } - } - if Instant::now() >= deadline { - return Err(miette::miette!( - "timed out waiting for compose child services {:?} in {}", - services, - artifact_root.display() - )); - } - sleep(Duration::from_millis(200)).await; - } -} - -fn walk_files(root: &Path) -> Result> { - let mut files = Vec::new(); - let mut stack = vec![root.to_path_buf()]; - while let Some(path) = stack.pop() { - for entry in fs::read_dir(&path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to list {}", path.display()))? - { - let entry = entry.into_diagnostic()?; - let entry_path = entry.path(); - if entry_path.is_dir() { - stack.push(entry_path); - } else { - files.push(entry_path); - } - } - } - Ok(files) -} - -fn observability_endpoint_for_site(kind: SiteKind, endpoint: &str) -> Result { - if !matches!(kind, SiteKind::Compose | SiteKind::Kubernetes) { - return Ok(endpoint.to_string()); - } - - let mut url = Url::parse(endpoint) - .into_diagnostic() - .wrap_err_with(|| format!("invalid observability endpoint {endpoint}"))?; - let should_rewrite = url.host_str().is_some_and(|host| { - host.eq_ignore_ascii_case("localhost") - || host - .parse::() - .map(|addr| addr.is_loopback() || addr.is_unspecified()) - .unwrap_or(false) - }); - if should_rewrite { - let host = container_host_for_consumer(SiteKind::Direct, kind); - url.set_host(Some(&host)) - .map_err(|_| miette::miette!("failed to rewrite observability endpoint {endpoint}"))?; - } - Ok(url.to_string()) -} - -pub(crate) fn reserve_loopback_port() -> Result { - let listener = TcpListener::bind(("127.0.0.1", 0)) - .into_diagnostic() - .wrap_err("failed to allocate a loopback port")?; - Ok(listener.local_addr().into_diagnostic()?.port()) -} - -pub(crate) fn new_run_id() -> String { - let millis = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_millis(); - format!("run-{millis:x}-{:x}", std::process::id()) -} - -fn receipt_path(run_root: &Path) -> PathBuf { - run_root.join("receipt.json") -} - -fn run_plan_path(run_root: &Path) -> PathBuf { - run_root.join("run-plan.json") -} - -fn launch_bundle_manifest_path(run_root: &Path) -> PathBuf { - run_root.join("launch-bundle.json") -} - -pub(crate) fn site_state_path(state_root: &Path, site_id: &str) -> PathBuf { - state_root.join(site_id).join("manager-state.json") -} - -fn state_path_for_plan(plan: &SiteSupervisorPlan) -> PathBuf { - Path::new(&plan.site_state_root).join("manager-state.json") -} - -fn site_supervisor_plan_path(site_state_root: &Path) -> PathBuf { - site_state_root.join("site-supervisor-plan.json") -} - -pub(crate) fn site_controller_plan_path(site_state_root: &Path) -> PathBuf { - site_state_root.join("site-controller-plan.json") -} - -pub(crate) fn desired_links_path(site_state_root: &Path) -> PathBuf { - site_state_root.join("desired-links.json") -} - -fn commit_marker_path(run_root: &Path) -> PathBuf { - run_root.join("committed") -} - -fn coordinator_lock_path(run_root: &Path) -> PathBuf { - run_root.join("coordinator.lock") -} - -fn stop_marker_path(run_root: &Path) -> PathBuf { - run_root.join("stop-requested") -} - -async fn wait_for_stop_request(stop_requested: &AtomicBool, run_root: &Path) { - loop { - if stop_requested.load(Ordering::SeqCst) || stop_marker_path(run_root).exists() { - return; - } - sleep(Duration::from_millis(100)).await; - } -} - -async fn run_until_stop( - run_root: &Path, - stop_requested: &AtomicBool, - future: F, -) -> Result> -where - F: Future>, -{ - tokio::pin!(future); - tokio::select! { - result = &mut future => result.map(Some), - _ = wait_for_stop_request(stop_requested, run_root) => Ok(None), - } -} - -async fn router_mesh_listener_ready(addr: SocketAddr) -> bool { - matches!( - tokio::time::timeout( - Duration::from_millis(250), - tokio::net::TcpStream::connect(addr) - ) - .await, - Ok(Ok(_)) - ) -} - -fn observability_plan_path(run_root: &Path) -> PathBuf { - run_root.join("observability").join("sink-plan.json") -} - -fn observability_state_path(run_root: &Path) -> PathBuf { - run_root.join("observability").join("sink-state.json") -} - -fn outside_proxy_plan_path(run_root: &Path) -> PathBuf { - run_root.join("outside-proxy-plan.json") -} - -fn outside_proxy_state_path(run_root: &Path) -> PathBuf { - run_root.join("outside-proxy-state.json") -} - -fn write_commit_marker(run_root: &Path) -> Result<()> { - fs::write(commit_marker_path(run_root), b"committed") - .into_diagnostic() - .wrap_err("failed to write commit marker") -} - -fn write_stop_marker(run_root: &Path) -> Result<()> { - fs::write(stop_marker_path(run_root), b"stop") - .into_diagnostic() - .wrap_err("failed to write stop marker") + .wrap_err("failed to write stop marker") } fn test_wave_delay() -> Result> { @@ -7470,24 +1374,6 @@ fn send_sigterm(pid: u32) { } } -#[cfg(unix)] -fn send_signal_to_pids(pids: &[u32], signal: i32) { - for pid in pids { - let Some(pid) = i32::try_from(*pid).ok() else { - continue; - }; - let _ = unsafe { libc::kill(pid, signal) }; - } -} - -#[cfg(unix)] -fn send_signal_to_process_group(root_pid: u32, signal: i32) { - let Some(root_pid) = i32::try_from(root_pid).ok() else { - return; - }; - let _ = unsafe { libc::kill(-root_pid, signal) }; -} - fn install_signal_flag() -> Arc { let flag = Arc::new(AtomicBool::new(false)); @@ -7556,15 +1442,7 @@ fn emit_manager_event(event_name: &'static str, body: String, attributes: &[(&st let _ = send_manager_observability(&endpoint, "/v1/logs", &payload); } -fn handle_otlp_connection( - stream: &mut TcpStream, - requests_log: &Path, - events_ndjson: &Path, -) -> Result<()> { - stream - .set_nonblocking(false) - .into_diagnostic() - .wrap_err("failed to configure observability stream blocking mode")?; +fn handle_otlp_connection(stream: &mut TcpStream, requests_log: &Path) -> Result<()> { stream .set_read_timeout(Some(Duration::from_secs(5))) .into_diagnostic() @@ -7595,15 +1473,6 @@ fn handle_otlp_connection( .and_then(|line| line.split_whitespace().nth(1)) .unwrap_or("/") .to_string(); - let content_type = header - .lines() - .find_map(|line| { - let (name, value) = line.split_once(':')?; - name.trim() - .eq_ignore_ascii_case("content-type") - .then_some(value.trim().to_ascii_lowercase()) - }) - .unwrap_or_default(); let content_length = header .lines() .find_map(|line| { @@ -7641,17 +1510,6 @@ fn handle_otlp_connection( writeln!(log, "{}\t{}", path, body_len) .into_diagnostic() .wrap_err_with(|| format!("failed to append {}", requests_log.display()))?; - if path == "/v1/logs" { - let body = &buf[body_start..]; - if content_type.contains("application/json") || body.starts_with(b"{") { - let _ = serde_json::from_slice::(body); - } else { - match persist_otlp_log_records(body) { - Ok(events) => append_jsonl(events_ndjson, &events)?, - Err(err) => eprintln!("warning: failed to decode OTLP log payload: {err}"), - } - } - } stream .write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") .into_diagnostic() @@ -7659,215 +1517,6 @@ fn handle_otlp_connection( Ok(()) } -fn append_jsonl(path: &Path, events: &[PersistedTraceEvent]) -> Result<()> { - if events.is_empty() { - return Ok(()); - } - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - let mut log = fs::OpenOptions::new() - .create(true) - .append(true) - .open(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to open {}", path.display()))?; - for event in events { - serde_json::to_writer(&mut log, event) - .into_diagnostic() - .wrap_err_with(|| format!("failed to append {}", path.display()))?; - writeln!(log) - .into_diagnostic() - .wrap_err_with(|| format!("failed to append {}", path.display()))?; - } - Ok(()) -} - -fn persist_otlp_log_records(body: &[u8]) -> Result> { - let request = ExportLogsServiceRequest::decode(body) - .into_diagnostic() - .wrap_err("invalid OTLP log payload")?; - let mut events = Vec::new(); - for resource_logs in &request.resource_logs { - let resource_attrs = resource_logs - .resource - .as_ref() - .map(|resource| resource.attributes.as_slice()) - .unwrap_or(&[]); - for scope_logs in &resource_logs.scope_logs { - for record in &scope_logs.log_records { - if let Some(event) = persist_otlp_log_record(resource_attrs, record) { - events.push(event); - } - } - } - } - Ok(events) -} - -fn persist_otlp_log_record( - resource_attrs: &[OtlpKeyValue], - record: &OtlpLogRecord, -) -> Option { - let entity_kind = otlp_attribute(resource_attrs, &record.attributes, "amber_entity_kind"); - let edge_ref = otlp_attribute(resource_attrs, &record.attributes, "amber_edge_ref"); - if entity_kind.as_deref() != Some("binding") { - return None; - } - if edge_ref - .as_deref() - .is_some_and(|edge_ref| edge_ref.starts_with("/site/")) - { - return None; - } - - let body = record.body.as_ref().map(otlp_any_value_to_text)?; - let body = body.trim(); - if body.is_empty() { - return None; - } - - Some(PersistedTraceEvent { - schema: PERSISTED_TRACE_EVENT_SCHEMA.to_string(), - version: PERSISTED_TRACE_EVENT_VERSION, - observed_at_unix_nano: nonzero_u64(record.time_unix_nano) - .or_else(|| nonzero_u64(record.observed_time_unix_nano)), - message: body.to_string(), - severity: nonempty(record.severity_text.trim()), - trace_id: otlp_trace_id(&record.trace_id), - attributes: merged_otlp_attributes(resource_attrs, &record.attributes), - }) -} - -fn merged_otlp_attributes( - resource_attrs: &[OtlpKeyValue], - record_attrs: &[OtlpKeyValue], -) -> BTreeMap { - let mut attributes = BTreeMap::new(); - for attr in resource_attrs { - if let Some(value) = attr.value.as_ref() { - attributes.insert(attr.key.clone(), otlp_any_value_to_json(value)); - } - } - for attr in record_attrs { - if let Some(value) = attr.value.as_ref() { - attributes.insert(attr.key.clone(), otlp_any_value_to_json(value)); - } - } - attributes -} - -fn nonzero_u64(value: u64) -> Option { - (value != 0).then_some(value) -} - -fn nonempty(value: &str) -> Option { - (!value.is_empty()).then_some(value.to_string()) -} - -fn otlp_attribute( - resource_attrs: &[OtlpKeyValue], - record_attrs: &[OtlpKeyValue], - key: &str, -) -> Option { - if let Some(value) = otlp_attribute_from(record_attrs, key) { - return Some(value); - } - otlp_attribute_from(resource_attrs, key) -} - -fn otlp_attribute_from(attrs: &[OtlpKeyValue], key: &str) -> Option { - attrs.iter().find_map(|attr| { - (attr.key == key) - .then(|| attr.value.as_ref().map(otlp_any_value_to_text)) - .flatten() - }) -} - -fn otlp_any_value_to_text(value: &OtlpAnyValue) -> String { - match value.value.as_ref() { - Some(any_value::Value::StringValue(value)) => value.clone(), - Some(any_value::Value::BoolValue(value)) => value.to_string(), - Some(any_value::Value::IntValue(value)) => value.to_string(), - Some(any_value::Value::DoubleValue(value)) => value.to_string(), - Some(any_value::Value::BytesValue(value)) => format!("0x{}", hex_bytes(value)), - Some(any_value::Value::ArrayValue(values)) => values - .values - .iter() - .map(otlp_any_value_to_text) - .collect::>() - .join(", "), - Some(any_value::Value::KvlistValue(values)) => values - .values - .iter() - .map(|entry| { - format!( - "{}={}", - entry.key, - entry - .value - .as_ref() - .map(otlp_any_value_to_text) - .unwrap_or_default() - ) - }) - .collect::>() - .join(", "), - None => String::new(), - } -} - -fn otlp_any_value_to_json(value: &OtlpAnyValue) -> JsonValue { - match value.value.as_ref() { - Some(any_value::Value::StringValue(value)) => JsonValue::String(value.clone()), - Some(any_value::Value::BoolValue(value)) => JsonValue::Bool(*value), - Some(any_value::Value::IntValue(value)) => JsonValue::Number((*value).into()), - Some(any_value::Value::DoubleValue(value)) => serde_json::Number::from_f64(*value) - .map(JsonValue::Number) - .unwrap_or(JsonValue::Null), - Some(any_value::Value::BytesValue(value)) => { - JsonValue::String(format!("0x{}", hex_bytes(value))) - } - Some(any_value::Value::ArrayValue(values)) => JsonValue::Array( - values - .values - .iter() - .map(otlp_any_value_to_json) - .collect::>(), - ), - Some(any_value::Value::KvlistValue(values)) => JsonValue::Object( - values - .values - .iter() - .filter_map(|entry| { - Some(( - entry.key.clone(), - otlp_any_value_to_json(entry.value.as_ref()?), - )) - }) - .collect::>(), - ), - None => JsonValue::Null, - } -} - -fn otlp_trace_id(trace_id: &[u8]) -> Option { - if trace_id.len() != 16 || trace_id.iter().all(|byte| *byte == 0) { - return None; - } - Some(hex_bytes(trace_id)) -} - -fn hex_bytes(bytes: &[u8]) -> String { - let mut out = String::with_capacity(bytes.len() * 2); - for byte in bytes { - write!(&mut out, "{byte:02x}").expect("writing to an in-memory string should not fail"); - } - out -} - #[cfg(unix)] fn send_sigkill(pid: u32) { let _ = unsafe { libc::kill(pid as i32, libc::SIGKILL) }; diff --git a/cli/src/mixed_run/launch_bundle.rs b/cli/src/mixed_run/launch_bundle.rs index 9bca52b0..0e5b98eb 100644 --- a/cli/src/mixed_run/launch_bundle.rs +++ b/cli/src/mixed_run/launch_bundle.rs @@ -1,5 +1,21 @@ use super::*; +pub(super) fn site_controller_image_reference_from_overrides( + overrides: &BTreeMap, +) -> String { + overrides + .get("site_controller") + .map(|tag| amber_images::override_reference(&amber_images::AMBER_SITE_CONTROLLER, tag)) + .unwrap_or_else(|| amber_images::AMBER_SITE_CONTROLLER.reference.to_string()) +} + +fn site_controller_image_reference() -> Result { + let overrides = + amber_images::parse_dev_image_tag_overrides(amber_images::INTERNAL_IMAGE_OVERRIDE_KEYS) + .map_err(|err| miette::miette!(err))?; + Ok(site_controller_image_reference_from_overrides(&overrides)) +} + pub(crate) fn dry_run_run_plan( source_plan_path: Option<&Path>, run_plan: &RunPlan, @@ -52,6 +68,7 @@ pub(super) fn materialize_launch_bundle( let run_plan_path = run_plan_path(bundle_root); write_json(&run_plan_path, run_plan)?; + let site_controller_image = site_controller_image_reference()?; let controller_auth_token = amber_site_controller::generate_framework_auth_token( &run_plan.mesh_scope, @@ -68,25 +85,6 @@ pub(super) fn materialize_launch_bundle( miette::miette!("site controller dynamic capability signing seed is invalid: {err}") })?, ); - let controller_endpoints = run_plan - .sites - .iter() - .map(|(site_id, site_plan)| { - let port = reserve_loopback_port()?; - let listen_addr = host_service_bind_addr_for_consumer(site_plan.site.kind, port); - let host = match site_plan.site.kind { - SiteKind::Direct | SiteKind::Vm => "127.0.0.1".to_string(), - SiteKind::Compose | SiteKind::Kubernetes => { - host_service_host_for_consumer(site_plan.site.kind) - } - }; - Ok(( - site_id.clone(), - (listen_addr, format!("http://{host}:{port}")), - )) - }) - .collect::>>()?; - let observability = materialize_observability(bundle_root, run_id, &run_plan.mesh_scope, observability)?; let observability_endpoint = observability @@ -96,13 +94,25 @@ pub(super) fn materialize_launch_bundle( let mut sites = BTreeMap::new(); for (site_index, (site_id, site_plan)) in run_plan.sites.iter().enumerate() { let artifact_dir = materialize_site_artifacts(&sites_root, site_id, site_plan)?; + amber_site_controller::set_site_artifact_mesh_identity_seed(&artifact_dir, run_id)?; let site_state_root = state_root.join(site_id); let controller_state_path = site_state_root.join("site-controller-state.json"); let controller_plan_path = site_controller_plan_path(&site_state_root); - let (controller_listen_addr, controller_url) = controller_endpoints - .get(site_id) - .cloned() - .expect("controller endpoint should exist for every site"); + let (controller_listen_addr, controller_url) = match site_plan.site.kind { + SiteKind::Direct | SiteKind::Vm => { + let port = reserve_loopback_port()?; + let listen_addr = host_service_bind_addr_for_consumer(site_plan.site.kind, port); + (listen_addr, format!("http://127.0.0.1:{port}")) + } + SiteKind::Compose | SiteKind::Kubernetes => ( + SocketAddr::from(([0, 0, 0, 0], amber_site_controller::SITE_CONTROLLER_PORT)), + format!( + "http://{}:{}", + amber_site_controller::SITE_CONTROLLER_SERVICE_NAME, + amber_site_controller::SITE_CONTROLLER_PORT + ), + ), + }; let mut framework_env = BTreeMap::new(); framework_env.insert( amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV.to_string(), @@ -142,11 +152,23 @@ pub(super) fn materialize_launch_bundle( artifact_dir: &artifact_dir, site_state_root: &site_state_root, observability_endpoint, - site_controller_plan_path: Some(controller_plan_path.as_path()), + site_controller_plan_path: matches!( + site_plan.site.kind, + SiteKind::Direct | SiteKind::Vm + ) + .then_some(controller_plan_path.as_path()), site_controller_url: Some(controller_url.as_str()), }, launch_env.clone(), )?; + if site_plan.site.kind == SiteKind::Compose + && let Some(router_mesh_port) = base_supervisor_plan.router_mesh_port + { + amber_site_controller::set_compose_router_published_mesh_port( + &artifact_dir, + router_mesh_port, + )?; + } let controller_state = amber_site_controller::build_site_controller_state( run_id, run_plan, @@ -156,61 +178,6 @@ pub(super) fn materialize_launch_bundle( &dynamic_capability_signing_seed_b64, )?; amber_site_controller::write_control_state(&controller_state_path, &controller_state)?; - let peer_controllers = controller_endpoints - .iter() - .filter(|(peer_site_id, _)| *peer_site_id != site_id) - .map(|(peer_site_id, (peer_listen_addr, _peer_url))| { - ( - peer_site_id.clone(), - amber_site_controller::SiteControllerPeerPlan { - site_id: peer_site_id.clone(), - kind: run_plan - .sites - .get(peer_site_id) - .expect("peer site should exist in run plan") - .site - .kind, - // Peer controllers talk to the host-side controller process directly. - // The site-facing authority URL may use container-oriented hostnames such - // as `host.docker.internal`, which are correct for workloads inside a - // site but wrong for another host-side controller process. - authority_url: amber_site_controller::authority_url_for_listen_addr( - *peer_listen_addr, - ), - }, - ) - }) - .collect(); - amber_site_controller::write_site_controller_plan( - &controller_plan_path, - run_id, - &run_plan.mesh_scope, - site_id, - site_plan.site.kind, - controller_listen_addr, - &controller_url, - &site_plan.router_identity_id, - &controller_state_path, - bundle_root, - &state_root, - &site_state_root, - &artifact_dir, - &controller_auth_token, - &dynamic_caps_token_verify_key_b64, - peer_controllers, - base_supervisor_plan.storage_root.as_deref(), - base_supervisor_plan.runtime_root.as_deref(), - base_supervisor_plan.router_mesh_port, - base_supervisor_plan.compose_project.as_deref(), - base_supervisor_plan.kubernetes_namespace.as_deref(), - base_supervisor_plan.context.as_deref(), - base_supervisor_plan.observability_endpoint.as_deref(), - &launch_env, - )?; - write_json( - &site_supervisor_plan_path(&site_state_root), - &base_supervisor_plan, - )?; write_json( &desired_links_path(&site_state_root), &DesiredLinkState { @@ -228,11 +195,138 @@ pub(super) fn materialize_launch_bundle( site_plan: site_plan.clone(), artifact_dir, site_state_root, + controller_state_path, + controller_plan_path, + controller_listen_addr, + controller_url, base_supervisor_plan, }, ); } + let planned_router_mesh_addrs = sites + .iter() + .map(|(site_id, site)| { + Ok(( + site_id.clone(), + planned_router_mesh_addr(&site.base_supervisor_plan)?, + )) + }) + .collect::>>()?; + + for (site_id, site) in &mut sites { + let allowed_issuers = run_plan + .sites + .iter() + .filter(|(peer_site_id, _)| *peer_site_id != site_id) + .map(|(_, peer_site)| peer_site.router_identity_id.clone()) + .collect::>(); + let route_listen_addr = + site_controller_route_listen_addr(site.site_plan.site.kind).to_string(); + let mut peer_site_router_urls = BTreeMap::new(); + let mut peer_router_mesh_addrs = BTreeMap::new(); + let mut controller_route_ports = Vec::new(); + let mut controller_routes = Vec::new(); + let peer_router_host = host_service_host_for_consumer(site.site_plan.site.kind); + for (peer_site_id, peer_site_plan) in &run_plan.sites { + if peer_site_id == site_id { + continue; + } + let route_port = reserve_loopback_port()?; + controller_route_ports.push(route_port); + peer_site_router_urls.insert( + peer_site_id.clone(), + format!("http://{peer_router_host}:{route_port}"), + ); + let peer_router_mesh_addr = planned_router_mesh_addrs + .get(peer_site_id) + .expect("planned router mesh addr should exist for every site"); + peer_router_mesh_addrs.insert(peer_site_id.clone(), peer_router_mesh_addr.clone()); + let peer_addr = amber_site_controller::router_mesh_addr_for_consumer( + peer_site_plan.site.kind, + site.site_plan.site.kind, + peer_router_mesh_addr, + )?; + controller_routes.push(amber_site_controller::SiteControllerPeerRouterRoute { + site_id: peer_site_id.clone(), + peer_router: planned_router_identity( + run_id, + &run_plan.mesh_scope, + &peer_site_plan.router_identity_id, + ), + peer_addr, + listen_addr: route_listen_addr.clone(), + listen_port: route_port, + }); + } + + amber_site_controller::inject_site_controller_peer_router_routes( + &site.artifact_dir, + site_id, + &allowed_issuers, + &controller_routes, + )?; + if site.site_plan.site.kind == SiteKind::Compose { + amber_site_controller::add_compose_router_published_route_ports( + &site.artifact_dir, + &controller_route_ports, + )?; + } + site.base_supervisor_plan.controller_route_ports = controller_route_ports; + + amber_site_controller::write_site_controller_plan( + &site.controller_plan_path, + run_id, + &run_plan.mesh_scope, + site_id, + site.site_plan.site.kind, + site.controller_listen_addr, + &site.controller_url, + &site.site_plan.router_identity_id, + &peer_site_router_urls, + &peer_router_mesh_addrs, + match site.site_plan.site.kind { + SiteKind::Compose => Some("unix:///amber/control/router-control.sock"), + SiteKind::Kubernetes => Some("amber-router:24100"), + SiteKind::Direct | SiteKind::Vm => None, + }, + planned_router_mesh_addrs.get(site_id).map(String::as_str), + &site.controller_state_path, + bundle_root, + &state_root, + &site.site_state_root, + &site.artifact_dir, + &controller_auth_token, + &dynamic_caps_token_verify_key_b64, + site.base_supervisor_plan.storage_root.as_deref(), + site.base_supervisor_plan.runtime_root.as_deref(), + site.base_supervisor_plan.router_mesh_port, + site.base_supervisor_plan.compose_project.as_deref(), + site.base_supervisor_plan.kubernetes_namespace.as_deref(), + site.base_supervisor_plan.context.as_deref(), + site.base_supervisor_plan.observability_endpoint.as_deref(), + &site.base_supervisor_plan.launch_env, + )?; + match site.site_plan.site.kind { + SiteKind::Compose => amber_site_controller::inject_compose_site_controller( + &site.artifact_dir, + &read_json(&site.controller_plan_path, "site controller plan")?, + &site.controller_plan_path, + site_controller_image.as_str(), + )?, + SiteKind::Kubernetes => amber_site_controller::inject_kubernetes_site_controller( + &site.artifact_dir, + &read_json(&site.controller_plan_path, "site controller plan")?, + site_controller_image.as_str(), + )?, + SiteKind::Direct | SiteKind::Vm => {} + } + write_json( + &site_supervisor_plan_path(&site.site_state_root), + &site.base_supervisor_plan, + )?; + } + let manifest = build_launch_bundle_manifest( run_id, source_plan_path, @@ -251,6 +345,45 @@ pub(super) fn materialize_launch_bundle( }) } +fn planned_router_mesh_addr(plan: &SiteSupervisorPlan) -> Result { + match plan.kind { + SiteKind::Direct | SiteKind::Vm | SiteKind::Compose => plan + .router_mesh_port + .map(|port| format!("127.0.0.1:{port}")) + .ok_or_else(|| { + miette::miette!("site `{}` is missing its router mesh port", plan.site_id) + }), + SiteKind::Kubernetes => plan + .port_forward_mesh_port + .map(|port| format!("127.0.0.1:{port}")) + .ok_or_else(|| { + miette::miette!( + "site `{}` is missing its kubernetes router mesh forward port", + plan.site_id + ) + }), + } +} + +fn site_controller_route_listen_addr(kind: SiteKind) -> &'static str { + match kind { + SiteKind::Direct | SiteKind::Vm => "127.0.0.1", + SiteKind::Compose | SiteKind::Kubernetes => "0.0.0.0", + } +} + +fn planned_router_identity( + identity_seed: &str, + mesh_scope: &str, + router_identity_id: &str, +) -> amber_mesh::MeshIdentityPublic { + amber_mesh::MeshIdentityPublic::from_identity(&amber_mesh::MeshIdentity::derive( + router_identity_id, + Some(mesh_scope.to_string()), + identity_seed, + )) +} + pub(super) fn build_launch_bundle_manifest( run_id: &str, source_plan_path: Option<&Path>, @@ -367,7 +500,7 @@ pub(super) fn site_launch_preview(plan: &SiteSupervisorPlan) -> Result { diff --git a/cli/src/mixed_run/supervisor.rs b/cli/src/mixed_run/supervisor.rs index 91c98c2d..437ea738 100644 --- a/cli/src/mixed_run/supervisor.rs +++ b/cli/src/mixed_run/supervisor.rs @@ -228,7 +228,10 @@ pub(super) fn build_supervisor_plan( .then(|| input.site_state_root.join("storage").display().to_string()), runtime_root: matches!(input.site_plan.site.kind, SiteKind::Direct | SiteKind::Vm) .then(|| input.site_state_root.join("runtime").display().to_string()), - router_mesh_port: if matches!(input.site_plan.site.kind, SiteKind::Direct | SiteKind::Vm) { + router_mesh_port: if matches!( + input.site_plan.site.kind, + SiteKind::Direct | SiteKind::Vm | SiteKind::Compose + ) { Some(reserve_loopback_port()?) } else { None @@ -260,6 +263,7 @@ pub(super) fn build_supervisor_plan( .site_controller_plan_path .map(|path| path.display().to_string()), site_controller_url: input.site_controller_url.map(ToOwned::to_owned), + controller_route_ports: Vec::new(), launch_env, }) } @@ -526,68 +530,6 @@ pub(crate) fn update_desired_links_for_provider( write_json(&path, &state) } -pub(crate) fn update_desired_overlay_for_consumer( - site_state_root: &Path, - overlay_id: &str, - overlay: DesiredExternalSlotOverlay, -) -> Result<()> { - let path = desired_links_path(site_state_root); - let mut state: DesiredLinkState = if path.is_file() { - read_json(&path, "desired links")? - } else { - empty_desired_link_state() - }; - state - .external_slot_overlays - .insert(overlay_id.to_string(), overlay); - write_json(&path, &state) -} - -pub(crate) fn update_desired_overlay_for_provider( - site_state_root: &Path, - overlay_id: &str, - overlay: DesiredExportPeerOverlay, -) -> Result<()> { - let path = desired_links_path(site_state_root); - let mut state: DesiredLinkState = if path.is_file() { - read_json(&path, "desired links")? - } else { - empty_desired_link_state() - }; - state - .export_peer_overlays - .insert(overlay_id.to_string(), overlay); - write_json(&path, &state) -} - -pub(crate) fn clear_desired_overlay_for_consumer( - site_state_root: &Path, - overlay_id: &str, -) -> Result<()> { - let path = desired_links_path(site_state_root); - let mut state: DesiredLinkState = if path.is_file() { - read_json(&path, "desired links")? - } else { - return Ok(()); - }; - state.external_slot_overlays.remove(overlay_id); - write_json(&path, &state) -} - -pub(crate) fn clear_desired_overlay_for_provider( - site_state_root: &Path, - overlay_id: &str, -) -> Result<()> { - let path = desired_links_path(site_state_root); - let mut state: DesiredLinkState = if path.is_file() { - read_json(&path, "desired links")? - } else { - return Ok(()); - }; - state.export_peer_overlays.remove(overlay_id); - write_json(&path, &state) -} - fn empty_desired_link_state() -> DesiredLinkState { DesiredLinkState { schema: DESIRED_LINKS_SCHEMA.to_string(), @@ -699,7 +641,8 @@ pub(super) async fn ensure_site_running( reap_child(&mut runtime.port_forward)?; reap_child(&mut runtime.site_controller)?; - if runtime.site_controller.is_none() + if matches!(plan.kind, SiteKind::Direct | SiteKind::Vm) + && runtime.site_controller.is_none() && let Some(plan_path) = plan.site_controller_plan_path.as_deref() { let controller = super::site_controller_command()?; @@ -1717,9 +1660,11 @@ pub(super) fn spawn_port_forward(plan: &SiteSupervisorPlan) -> Result { .arg("0.0.0.0") .arg("deploy/amber-router") .arg(format!("{mesh_port}:24000")) - .arg(format!("{control_port}:24100")) - .stdout(Stdio::from(log)) - .stderr(Stdio::from(log_err)); + .arg(format!("{control_port}:24100")); + for port in &plan.controller_route_ports { + cmd.arg(format!("{port}:{port}")); + } + cmd.stdout(Stdio::from(log)).stderr(Stdio::from(log_err)); cmd.spawn() .into_diagnostic() .wrap_err("failed to spawn kubectl port-forward") @@ -1951,37 +1896,6 @@ pub(crate) fn host_service_bind_addr_for_consumer( host_proxy_bind_addr(consumer_needs_host_wide_listener(consumer_kind), port) } -pub(crate) fn router_mesh_addr_for_consumer( - provider_kind: SiteKind, - consumer_kind: SiteKind, - router_mesh_addr: &str, -) -> Result { - match consumer_kind { - SiteKind::Compose | SiteKind::Kubernetes => { - let addr = router_mesh_addr - .parse::() - .into_diagnostic() - .wrap_err_with(|| { - format!("invalid live router mesh address `{router_mesh_addr}`") - })?; - let host = container_host_for_consumer(provider_kind, consumer_kind); - Ok(format!("{host}:{}", addr.port())) - } - SiteKind::Direct | SiteKind::Vm => { - #[cfg(target_os = "linux")] - { - Ok(crate::direct_runtime::rewrite_peer_addr_for_slirp_gateway( - router_mesh_addr, - )) - } - #[cfg(not(target_os = "linux"))] - { - Ok(router_mesh_addr.to_string()) - } - } - } -} - pub(super) fn host_proxy_bind_addr(needs_host_wide_listener: bool, port: u16) -> SocketAddr { if needs_host_wide_listener { SocketAddr::from(([0, 0, 0, 0], port)) diff --git a/cli/src/mixed_run/tests.rs b/cli/src/mixed_run/tests.rs index 848dbd27..0d5700fe 100644 --- a/cli/src/mixed_run/tests.rs +++ b/cli/src/mixed_run/tests.rs @@ -1,4126 +1,36 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - io::{Read as _, Write as _}, - net::{TcpListener, TcpStream}, - sync::Mutex, -}; - -use amber_compiler::reporter::direct::{ - DIRECT_PLAN_VERSION, DirectComponentPlan, DirectPlan, DirectProgramExecutionPlan, - DirectProgramPlan, DirectRuntimeAddressPlan, DirectSidecarPlan, -}; -use amber_site_controller::DynamicProxyExportRecord; -use base64::Engine as _; -use opentelemetry_proto::tonic::{ - collector::logs::v1::ExportLogsServiceRequest, - common::v1::{AnyValue, InstrumentationScope, KeyValue, any_value}, - logs::v1::{LogRecord, ResourceLogs, ScopeLogs}, - resource::v1::Resource, -}; -use prost::Message as _; -use tempfile::tempdir; -use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; - use super::*; -static EXECUTABLE_ENV_LOCK: Mutex<()> = Mutex::new(()); - -fn test_dynamic_proxy_export_record( - component_id: usize, - component: &str, - provide: &str, - protocol: &str, - capability_kind: &str, - target_port: u16, -) -> DynamicProxyExportRecord { - DynamicProxyExportRecord { - component_id, - component: component.to_string(), - provide: provide.to_string(), - protocol: protocol.to_string(), - capability_kind: capability_kind.to_string(), - capability_profile: None, - target_port, - } -} - -fn test_site_receipt( - kind: SiteKind, - artifact_dir: &Path, - router_control: Option<&str>, - router_mesh_addr: Option<&str>, -) -> SiteReceipt { - SiteReceipt { - kind, - artifact_dir: artifact_dir.display().to_string(), - supervisor_pid: 100, - process_pid: None, - compose_project: None, - kubernetes_namespace: None, - port_forward_pid: None, - context: None, - router_control: router_control.map(str::to_string), - router_mesh_addr: router_mesh_addr.map(str::to_string), - router_identity_id: None, - router_public_key_b64: None, - site_controller_pid: None, - site_controller_url: None, - } -} - -fn otlp_string_value(value: &str) -> AnyValue { - AnyValue { - value: Some(any_value::Value::StringValue(value.to_string())), - } -} - -#[test] -fn compose_start_failure_explains_missing_docker_daemon() { - let output = std::process::Command::new("sh") - .arg("-c") - .arg( - "echo 'Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the \ - docker daemon running?' 1>&2; exit 1", - ) - .output() - .expect("shell output"); - - let err = compose_start_failure("compose_local", &output); - let message = err.to_string(); - assert!(message.contains("could not reach Docker")); - assert!(message.contains("Start Docker Desktop or the Docker daemon")); - assert!(message.contains("Cannot connect to the Docker daemon")); -} - -#[test] -fn compose_start_failure_surfaces_specific_compose_stderr() { - let output = std::process::Command::new("sh") - .arg("-c") - .arg( - "echo 'Network amber_boundary Creating' 1>&2; echo 'services.web.image must be a \ - string' 1>&2; exit 1", - ) - .output() - .expect("shell output"); - - let err = compose_start_failure("compose_local", &output); - let message = err.to_string(); - assert!(message.contains("compose site `compose_local` failed to start")); - assert!(message.contains("services.web.image must be a string")); - assert!(!message.contains("Network amber_boundary Creating")); -} - -fn test_site_state( - run_id: &str, - site_id: &str, - kind: SiteKind, - artifact_dir: &Path, - router_control: Option<&str>, - router_mesh_addr: Option<&str>, -) -> SiteManagerState { - SiteManagerState { - schema: "amber.run.site-state".to_string(), - version: 1, - run_id: run_id.to_string(), - site_id: site_id.to_string(), - kind, - status: SiteLifecycleStatus::Running, - artifact_dir: artifact_dir.display().to_string(), - supervisor_pid: 101, - process_pid: None, - compose_project: None, - kubernetes_namespace: None, - port_forward_pid: None, - context: None, - router_control: router_control.map(str::to_string), - router_mesh_addr: router_mesh_addr.map(str::to_string), - router_identity_id: None, - router_public_key_b64: None, - site_controller_pid: None, - site_controller_url: None, - last_error: None, - } -} - -fn test_site_controller_runtime_plan( - kind: SiteKind, - artifact_dir: &Path, - site_state_root: &Path, -) -> SiteControllerRuntimePlan { - SiteControllerRuntimePlan { - schema: "amber.run.site_controller_runtime_plan".to_string(), - version: 1, - run_id: "run-test".to_string(), - mesh_scope: "test-scope".to_string(), - run_root: site_state_root.display().to_string(), - site_id: "direct_local".to_string(), - kind, - router_identity_id: "/site/direct_local/router".to_string(), - artifact_dir: artifact_dir.display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: None, - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), - } -} - -#[test] -fn forwarded_endpoint_ready_accepts_open_connection() { - let listener = - TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).expect("listener should bind"); - let addr = listener.local_addr().expect("listener addr"); - let handle = std::thread::spawn(move || { - let (_stream, _) = listener.accept().expect("listener should accept"); - std::thread::sleep(Duration::from_millis(500)); - }); - - assert!(crate::tcp_readiness::endpoint_accepts_stable_connection( - addr, - Duration::from_millis(250), - Duration::from_millis(250), - )); - handle.join().expect("listener thread should finish"); -} - -#[test] -fn forwarded_endpoint_ready_rejects_reset_connection() { - let listener = - TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).expect("listener should bind"); - let addr = listener.local_addr().expect("listener addr"); - let handle = std::thread::spawn(move || { - let (stream, _) = listener.accept().expect("listener should accept"); - drop(stream); - }); - - assert!(!crate::tcp_readiness::endpoint_accepts_stable_connection( - addr, - Duration::from_millis(250), - Duration::from_millis(250), - )); - handle.join().expect("listener thread should finish"); -} - -#[test] -fn desired_link_overlays_are_owned_per_overlay_id() { - let temp = tempdir().expect("tempdir"); - let site_state_root = temp.path(); - - super::supervisor::update_desired_overlay_for_consumer( - site_state_root, - "overlay-a", - DesiredExternalSlotOverlay { - slot_name: "api".to_string(), - url: "http://provider".to_string(), - }, - ) - .expect("first consumer overlay should persist"); - super::supervisor::update_desired_overlay_for_consumer( - site_state_root, - "overlay-b", - DesiredExternalSlotOverlay { - slot_name: "api".to_string(), - url: "http://provider".to_string(), - }, - ) - .expect("second consumer overlay should persist"); - super::supervisor::update_desired_overlay_for_provider( - site_state_root, - "provider-a", - DesiredExportPeerOverlay { - export_name: "amber_export_shared".to_string(), - peer_id: "consumer-a".to_string(), - peer_key_b64: "a2V5".to_string(), - protocol: "http".to_string(), - route_id: Some("route-a".to_string()), - }, - ) - .expect("first provider overlay should persist"); - super::supervisor::update_desired_overlay_for_provider( - site_state_root, - "provider-b", - DesiredExportPeerOverlay { - export_name: "amber_export_shared".to_string(), - peer_id: "consumer-b".to_string(), - peer_key_b64: "a2V5".to_string(), - protocol: "http".to_string(), - route_id: Some("route-b".to_string()), - }, - ) - .expect("second provider overlay should persist"); - - let path = desired_links_path(site_state_root); - let desired: DesiredLinkState = read_json(&path, "desired links").expect("desired links"); - assert_eq!(desired.external_slot_overlays.len(), 2); - assert_eq!(desired.export_peer_overlays.len(), 2); - - super::supervisor::clear_desired_overlay_for_consumer(site_state_root, "overlay-a") - .expect("consumer overlay removal should persist"); - super::supervisor::clear_desired_overlay_for_provider(site_state_root, "provider-a") - .expect("provider overlay removal should persist"); - - let desired: DesiredLinkState = read_json(&path, "desired links").expect("desired links"); - assert_eq!(desired.external_slot_overlays.len(), 1); - assert_eq!(desired.export_peer_overlays.len(), 1); - assert!(desired.external_slot_overlays.contains_key("overlay-b")); - assert!(desired.export_peer_overlays.contains_key("provider-b")); -} - -#[test] -fn site_router_control_endpoint_prefers_manager_state_for_local_sites() { - let temp = tempdir().expect("tempdir"); - let artifact_dir = temp.path().join("artifact"); - let site_state_root = temp.path().join("state"); - fs::create_dir_all(&artifact_dir).expect("artifact dir should exist"); - fs::create_dir_all(&site_state_root).expect("site state dir should exist"); - let manager_state_path = site_state_root.join("manager-state.json"); - - for kind in [SiteKind::Direct, SiteKind::Vm] { - write_json( - &manager_state_path, - &test_site_state( - "run-test", - "direct_local", - kind, - &artifact_dir, - Some("unix:///tmp/router-from-manager.sock"), - Some("127.0.0.1:24000"), - ), - ) - .expect("manager state should be written"); - - let endpoint = site_router_control_endpoint(&test_site_controller_runtime_plan( - kind, - &artifact_dir, - &site_state_root, - )) - .expect("router control endpoint should resolve"); - - assert_eq!( - endpoint.to_string(), - "unix:///tmp/router-from-manager.sock", - "local site controller runtime should use the live manager-state endpoint for {kind:?}", - ); - } -} - -fn test_local_mesh_config(path: &Path, protocol: MeshProtocol, port: u16) -> Result<()> { - write_json( - path, - &MeshConfigPublic { - identity: MeshIdentityPublic { - id: "/site/test/router".to_string(), - public_key: [7; 32], - mesh_scope: Some("test-scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 24000)), - control_listen: Some(SocketAddr::from(([127, 0, 0, 1], 24100))), - dynamic_caps_listen: None, - control_allow: None, - peers: Vec::new(), - inbound: vec![InboundRoute { - route_id: "route".to_string(), - capability: "http".to_string(), - capability_kind: None, - capability_profile: None, - protocol, - http_plugins: Vec::new(), - target: InboundTarget::Local { port }, - allowed_issuers: Vec::new(), - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - ) -} - -#[test] -fn mesh_config_local_targets_ready_accepts_http_inbound_routes() { - let temp = tempdir().expect("tempdir should be created"); - let config_path = temp.path().join("mesh-config.json"); - let listener = - TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).expect("listener should bind"); - let addr = listener.local_addr().expect("listener addr"); - test_local_mesh_config(&config_path, MeshProtocol::Http, addr.port()) - .expect("mesh config should be written"); - let handle = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("listener should accept"); - let mut request = [0u8; 256]; - let _ = stream.read(&mut request); - let _ = stream.write_all( - b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", - ); - }); - - assert!( - mesh_config_local_targets_ready(&config_path, Duration::from_secs(1)) - .expect("mesh config should be readable") - ); - handle.join().expect("listener thread should finish"); -} - -#[test] -fn mesh_config_local_targets_ready_rejects_unreachable_http_inbound_routes() { - let temp = tempdir().expect("tempdir should be created"); - let config_path = temp.path().join("mesh-config.json"); - let listener = - TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).expect("listener should bind"); - let addr = listener.local_addr().expect("listener addr"); - drop(listener); - test_local_mesh_config(&config_path, MeshProtocol::Http, addr.port()) - .expect("mesh config should be written"); - - assert!( - !mesh_config_local_targets_ready(&config_path, Duration::from_millis(100)) - .expect("mesh config should be readable") - ); -} - -#[test] -fn direct_peer_ports_for_artifact_reads_mesh_listen_ports_without_runtime_state() { - let temp = tempdir().expect("tempdir should be created"); - let artifact_root = temp.path().join("artifact"); - let runtime_root = temp.path().join("runtime"); - let mesh_config_rel = PathBuf::from("mesh/components/app/mesh-config.json"); - let mesh_config_path = runtime_root.join(&mesh_config_rel); - fs::create_dir_all( - mesh_config_path - .parent() - .expect("mesh config should have a parent directory"), - ) - .expect("mesh config directory should be created"); - fs::create_dir_all(artifact_root.join(".amber")).expect("artifact .amber dir should exist"); - write_json( - &mesh_config_path, - &MeshConfigPublic { - identity: MeshIdentityPublic { - id: "/app".to_string(), - public_key: [9; 32], - mesh_scope: None, - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 18081)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: Vec::new(), - inbound: Vec::new(), - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - ) - .expect("component mesh config should be written"); - write_json( - &artifact_root.join("direct-plan.json"), - &DirectPlan { - version: DIRECT_PLAN_VERSION.to_string(), - mesh_provision_plan: "{}".to_string(), - startup_order: vec![7], - components: vec![DirectComponentPlan { - id: 7, - moniker: "/app".to_string(), - log_name: "app".to_string(), - source_dir: None, - depends_on: Vec::new(), - sidecar: DirectSidecarPlan { - log_name: "app-sidecar".to_string(), - mesh_port: 0, - mesh_config_path: mesh_config_rel.display().to_string(), - mesh_identity_path: "mesh/components/app/mesh-identity.json".to_string(), - env_passthrough: Vec::new(), - }, - program: DirectProgramPlan { - log_name: "app-program".to_string(), - work_dir: "work/components/app".to_string(), - storage_mounts: Vec::new(), - execution: DirectProgramExecutionPlan::Direct { - entrypoint: vec!["/bin/echo".to_string()], - env: BTreeMap::new(), - }, - }, - }], - runtime_addresses: DirectRuntimeAddressPlan { - slots_by_scope: BTreeMap::new(), - slot_items_by_scope: BTreeMap::new(), - }, - router: None, - }, - ) - .expect("direct plan should be written"); - - let ports = direct_peer_ports_for_artifact(&artifact_root, &runtime_root).expect("peer ports"); - assert_eq!(ports.get("/app"), Some(&18081)); -} - -#[test] -fn read_compose_launch_env_returns_saved_launch_env() { - let temp = tempdir().expect("tempdir should be created"); - let run_root = temp.path().join("run-root"); - let state_root = run_root.join("state").join("compose_local"); - write_json( - &site_supervisor_plan_path(&state_root), - &SiteSupervisorPlan { - schema: SITE_PLAN_SCHEMA.to_string(), - version: SITE_PLAN_VERSION, - run_id: "run-123".to_string(), - mesh_scope: "test.scope".to_string(), - run_root: run_root.display().to_string(), - coordinator_pid: 1, - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - artifact_dir: temp.path().join("artifact").display().to_string(), - site_state_root: state_root.display().to_string(), - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: Some("amber-test".to_string()), - kubernetes_namespace: None, - context: None, - port_forward_mesh_port: None, - port_forward_control_port: None, - observability_endpoint: None, - site_controller_plan_path: None, - site_controller_url: None, - launch_env: BTreeMap::from([ - ("AMBER_CONFIG_TENANT".to_string(), "acme-local".to_string()), - ( - "AMBER_CONFIG_CATALOG_TOKEN".to_string(), - "demo-token".to_string(), - ), - ]), - }, - ) - .expect("site supervisor plan should be written"); - - assert_eq!( - read_compose_launch_env(&run_root, "compose_local") - .expect("compose launch env should be readable"), - BTreeMap::from([ - ( - "AMBER_CONFIG_CATALOG_TOKEN".to_string(), - "demo-token".to_string() - ), - ("AMBER_CONFIG_TENANT".to_string(), "acme-local".to_string()), - ]) - ); -} - -#[test] -fn site_launch_commands_include_site_controller_binary_when_present() { - let _guard = EXECUTABLE_ENV_LOCK - .lock() - .unwrap_or_else(|err| err.into_inner()); - let temp = tempdir().expect("tempdir should be created"); - let amber = temp.path().join("amber"); - let controller = temp.path().join("amber-site-controller"); - fs::write(&amber, "").expect("amber binary placeholder should be written"); - fs::write(&controller, "").expect("site controller binary placeholder should be written"); - let previous_amber = std::env::var_os("CARGO_BIN_EXE_amber"); - let previous_controller = std::env::var_os("CARGO_BIN_EXE_amber-site-controller"); - // This test serializes access to process-global environment mutation with - // `EXECUTABLE_ENV_LOCK`, so these temporary overrides do not race other tests. - unsafe { - std::env::set_var("CARGO_BIN_EXE_amber", &amber); - std::env::set_var("CARGO_BIN_EXE_amber-site-controller", &controller); - } - - let plan = SiteSupervisorPlan { - schema: SITE_PLAN_SCHEMA.to_string(), - version: SITE_PLAN_VERSION, - run_id: "run-123".to_string(), - mesh_scope: "test.scope".to_string(), - run_root: temp.path().join("run-root").display().to_string(), - coordinator_pid: 1, - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - artifact_dir: temp.path().join("artifact").display().to_string(), - site_state_root: temp.path().join("state").display().to_string(), - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: Some("amber-test".to_string()), - kubernetes_namespace: None, - context: None, - port_forward_mesh_port: None, - port_forward_control_port: None, - observability_endpoint: None, - site_controller_plan_path: Some( - temp.path() - .join("state") - .join("site-controller-plan.json") - .display() - .to_string(), - ), - site_controller_url: Some("http://127.0.0.1:41000".to_string()), - launch_env: BTreeMap::from([("AMBER_TEST".to_string(), "1".to_string())]), - }; - - let commands = site_launch_commands(&plan).expect("launch commands should build"); - assert_eq!( - commands.len(), - 2, - "compose site should list controller and site commands" - ); - assert_eq!(commands[0].argv[0], controller.display().to_string()); - assert_eq!(commands[0].argv[1], "--plan"); - assert!( - commands[0].argv[2].ends_with("site-controller-plan.json"), - "controller command should point at the site controller plan" - ); - assert_eq!(commands[0].env, plan.launch_env); - assert_eq!(commands[1].argv[0], "docker"); - - match previous_amber { - Some(value) => unsafe { std::env::set_var("CARGO_BIN_EXE_amber", value) }, - None => unsafe { std::env::remove_var("CARGO_BIN_EXE_amber") }, - } - match previous_controller { - Some(value) => unsafe { std::env::set_var("CARGO_BIN_EXE_amber-site-controller", value) }, - None => unsafe { std::env::remove_var("CARGO_BIN_EXE_amber-site-controller") }, - } -} - -#[test] -fn site_controller_command_falls_back_to_amber_subcommand_without_standalone_binary() { - let _guard = EXECUTABLE_ENV_LOCK - .lock() - .unwrap_or_else(|err| err.into_inner()); - let temp = tempdir().expect("tempdir should be created"); - let amber = temp.path().join("amber"); - let current = temp.path().join("not-the-controller"); - fs::write(&amber, "").expect("amber binary placeholder should be written"); - let previous_amber = std::env::var_os("CARGO_BIN_EXE_amber"); - let previous_controller = std::env::var_os("CARGO_BIN_EXE_amber-site-controller"); - unsafe { - std::env::set_var("CARGO_BIN_EXE_amber", &amber); - std::env::remove_var("CARGO_BIN_EXE_amber-site-controller"); - } - - let command = site_controller_command_from(¤t) - .expect("site controller command should fall back to amber"); - assert_eq!(command.executable, amber); - assert_eq!(command.prefix_args, vec!["run-site-controller"]); - - match previous_amber { - Some(value) => unsafe { std::env::set_var("CARGO_BIN_EXE_amber", value) }, - None => unsafe { std::env::remove_var("CARGO_BIN_EXE_amber") }, - } - match previous_controller { - Some(value) => unsafe { std::env::set_var("CARGO_BIN_EXE_amber-site-controller", value) }, - None => unsafe { std::env::remove_var("CARGO_BIN_EXE_amber-site-controller") }, - } -} - -#[test] -fn rewrite_dynamic_proxy_metadata_updates_compose_x_amber_exports() { - let temp = tempdir().expect("tempdir should be created"); - let artifact_root = temp.path(); - fs::write( - artifact_root.join("compose.yaml"), - r#" -services: {} -x-amber: - version: "1" - router: - mesh_port: 24000 - control_port: 24100 - exports: - stale: - component: /stale - provide: old - protocol: http - router_mesh_port: 24000 -"#, - ) - .expect("compose artifact should be written"); - - rewrite_dynamic_proxy_metadata( - artifact_root, - &DynamicSitePlanRecord { - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - router_identity_id: "router".to_string(), - component_ids: Vec::new(), - assigned_components: Vec::new(), - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::from([( - "http".to_string(), - test_dynamic_proxy_export_record(8, "/job/root", "http", "http", "http", 8080), - )]), - routed_inputs: Vec::new(), - }, - ) - .expect("compose proxy metadata should rewrite"); - - let raw = fs::read_to_string(artifact_root.join("compose.yaml")) - .expect("compose artifact should be readable"); - let document: serde_yaml::Value = - serde_yaml::from_str(&raw).expect("compose yaml should remain valid"); - let exports = document["x-amber"]["exports"] - .as_mapping() - .expect("compose x-amber exports should be a mapping"); - assert_eq!(exports.len(), 1, "stale compose exports should be replaced"); - let export = exports - .get(serde_yaml::Value::String("http".to_string())) - .expect("expected rewritten compose export"); - assert_eq!(export["component"].as_str(), Some("/job/root")); - assert_eq!(export["provide"].as_str(), Some("http")); - assert_eq!(export["protocol"].as_str(), Some("http")); -} - -#[test] -fn prepare_dynamic_compose_child_artifact_keeps_only_child_owned_services() { - let temp = tempdir().expect("tempdir should be created"); - let site_artifact = temp.path().join("site-artifact"); - let child_artifact = temp.path().join("child-artifact"); - let site_state_root = temp.path().join("state").join("compose_local"); - let mesh_plan = MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![ - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/compose_admin".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([0, 0, 0, 0], 23000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/compose_local/router".to_string(), - }], - inbound: vec![InboundRoute { - route_id: component_route_id("/compose_admin", "http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 8080 }, - allowed_issuers: vec!["/site/compose_local/router".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "/amber/provision/c1-compose-admin-net".to_string(), - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/job/root".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([0, 0, 0, 0], 23000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/compose_local/router".to_string(), - }], - inbound: vec![InboundRoute { - route_id: component_route_id("/job/root", "http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 8080 }, - allowed_issuers: vec!["/site/compose_local/router".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "/amber/provision/c8-root-net".to_string(), - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Router, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/site/compose_local/router".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([0, 0, 0, 0], 24000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![ - amber_mesh::MeshPeerTemplate { - id: "/compose_admin".to_string(), - }, - amber_mesh::MeshPeerTemplate { - id: "/job/root".to_string(), - }, - ], - inbound: vec![ - InboundRoute { - route_id: router_export_route_id( - "compose_admin_http", - MeshProtocol::Http, - ), - capability: "compose_admin_http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::MeshForward { - peer_addr: "c1-compose-admin-net:23000".to_string(), - peer_id: "/compose_admin".to_string(), - route_id: component_route_id( - "/compose_admin", - "http", - MeshProtocol::Http, - ), - capability: "http".to_string(), - }, - allowed_issuers: vec!["/site/compose_local/router".to_string()], - }, - InboundRoute { - route_id: router_export_route_id("http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::MeshForward { - peer_addr: "c8-root-net:23000".to_string(), - peer_id: "/job/root".to_string(), - route_id: component_route_id( - "/job/root", - "http", - MeshProtocol::Http, - ), - capability: "http".to_string(), - }, - allowed_issuers: vec!["/site/compose_local/router".to_string()], - }, - InboundRoute { - route_id: "cap_static_admin".to_string(), - capability: "component".to_string(), - capability_kind: Some("component".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL".to_string(), - optional: false, - }, - allowed_issuers: vec!["/compose_admin".to_string()], - }, - InboundRoute { - route_id: "cap_child".to_string(), - capability: "component".to_string(), - capability_kind: Some("component".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL".to_string(), - optional: false, - }, - allowed_issuers: vec!["/job/root".to_string()], - }, - ], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "/amber/provision/amber-router".to_string(), - }, - }, - ], - }; - - fs::create_dir_all(&site_artifact).expect("site artifact dir should exist"); - fs::create_dir_all(&child_artifact).expect("child artifact dir should exist"); - fs::create_dir_all(&site_state_root).expect("site state dir should exist"); - - fs::write( - site_artifact.join("compose.yaml"), - r#" -services: - amber-router: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - amber-otelcol: - image: otel/opentelemetry-collector-contrib:0.143.0 - amber-router-control-init: - image: busybox:1.36.1 - c1-compose-admin: - image: python:3.13-alpine - depends_on: - - c1-compose-admin-net - c1-compose-admin-net: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - volumes: - - c1-compose-admin-net-mesh:/amber/mesh:ro -volumes: - c1-compose-admin-net-mesh: {} -x-amber: - version: "1" - router: - mesh_port: 24000 - control_port: 24100 -"#, - ) - .expect("static compose artifact should be written"); - - let mesh_plan_json = serde_json::to_string(&mesh_plan).expect("mesh plan should serialize"); - fs::write( - child_artifact.join("compose.yaml"), - format!( - r#" -services: - amber-router: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - amber-otelcol: - image: otel/opentelemetry-collector-contrib:0.143.0 - amber-router-control-init: - image: busybox:1.36.1 - amber-provisioner: - image: ghcr.io/rdi-foundation/amber-provisioner:v0.1 - c1-compose-admin: - image: python:3.13-alpine - depends_on: - - c1-compose-admin-net - c1-compose-admin-net: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - volumes: - - c1-compose-admin-net-mesh:/amber/mesh:ro - c8-root: - image: python:3.13-alpine - depends_on: - - c8-root-net - c8-root-net: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - depends_on: - - amber-provisioner - environment: - - AMBER_SCENARIO_SCOPE=scope - networks: - amber_mesh: {{}} - amber_egress_c8-root-net: {{}} - volumes: - - c8-root-net-mesh:/amber/mesh:ro -configs: - amber-mesh-provision-plan: - content: '{mesh_plan_json}' -volumes: - c1-compose-admin-net-mesh: {{}} - c8-root-net-mesh: {{}} -networks: - amber_mesh: - driver: bridge - internal: true - amber_egress_c8-root-net: - driver: bridge - internal: false -x-amber: - version: "1" - router: - mesh_port: 24000 - control_port: 24100 -"# - ), - ) - .expect("dynamic compose artifact should be written"); - - write_json( - &site_state_root.join("manager-state.json"), - &SiteManagerState { - schema: SITE_STATE_SCHEMA.to_string(), - version: SITE_STATE_VERSION, - run_id: "run-123".to_string(), - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - status: SiteLifecycleStatus::Running, - artifact_dir: site_artifact.display().to_string(), - supervisor_pid: 123, - process_pid: None, - compose_project: Some("amber-test".to_string()), - kubernetes_namespace: None, - port_forward_pid: None, - context: None, - router_control: Some("http://127.0.0.1:24100".to_string()), - router_mesh_addr: Some("127.0.0.1:24000".to_string()), - router_identity_id: Some("/site/compose_local/router".to_string()), - router_public_key_b64: Some( - base64::engine::general_purpose::STANDARD.encode([7u8; 32]), - ), - site_controller_pid: None, - site_controller_url: None, - last_error: None, - }, - ) - .expect("site manager state should be written"); - - supervisor::patch_site_artifacts( - &child_artifact, - "run-123", - "compose_local", - SiteKind::Compose, - &BTreeMap::new(), - None, - ) - .expect("compose child artifact should get explicit egress subnets"); - let mut patched_document = - read_compose_document(&child_artifact.join("compose.yaml")).expect("compose file"); - let initial_subnet = { - let patched_networks = - compose_networks_mut(&mut patched_document, &child_artifact.join("compose.yaml")) - .expect("compose networks") - .expect("compose networks should exist"); - assert!( - compose_network_subnet( - patched_networks - .get(yaml_string("amber_mesh")) - .expect("mesh network should exist"), - ) - .is_none(), - "mesh network should not get an egress subnet", - ); - compose_network_subnet( - patched_networks - .get(yaml_string("amber_egress_c8-root-net")) - .expect("child egress network should exist"), - ) - .expect("child egress network should get a subnet") - .to_string() - }; - supervisor::patch_site_artifacts( - &child_artifact, - "run-123", - "compose_local", - SiteKind::Compose, - &BTreeMap::new(), - None, - ) - .expect("compose subnet assignment should be idempotent"); - - prepare_dynamic_compose_child_artifact( - &SiteControllerRuntimePlan { - schema: SITE_PLAN_SCHEMA.to_string(), - version: SITE_PLAN_VERSION, - run_id: "run-123".to_string(), - mesh_scope: "mesh-scope-test".to_string(), - run_root: temp.path().display().to_string(), - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - router_identity_id: "/site/compose_local/router".to_string(), - artifact_dir: site_artifact.display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: Some("amber-test".to_string()), - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), - }, - &DynamicSitePlanRecord { - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - router_identity_id: "/site/compose_local/router".to_string(), - component_ids: vec![8], - assigned_components: vec!["/job/root".to_string()], - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), - }, - &child_artifact, - &[], - &BTreeMap::from([( - "/site/compose_local/router".to_string(), - MeshIdentityPublic { - id: "/site/compose_local/router".to_string(), - public_key: [7; 32], - mesh_scope: Some("mesh-scope-test".to_string()), - }, - )]), - ) - .expect("dynamic compose child artifact should be prepared"); - - let child_document = - read_compose_document(&child_artifact.join("compose.yaml")).expect("compose file"); - let child_services = - compose_services(&child_document, &child_artifact.join("compose.yaml")).expect("services"); - assert_eq!( - child_services - .keys() - .filter_map(serde_yaml::Value::as_str) - .collect::>(), - vec!["c8-root", "c8-root-net"] - ); - - let sidecar_volumes = child_services - .get(yaml_string("c8-root-net")) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|service| service.get(yaml_string("volumes"))) - .and_then(serde_yaml::Value::as_sequence) - .expect("child sidecar volumes should be present"); - assert_eq!( - sidecar_volumes, - &vec![serde_yaml::Value::String( - "./.amber/mesh/c8-root-net:/amber/mesh:ro".to_string() - )] - ); - let child_compose_yaml = fs::read_to_string(child_artifact.join("compose.yaml")) - .expect("child compose artifact should be readable"); - assert!( - child_compose_yaml.contains("AMBER_SCENARIO_SCOPE=mesh-scope-test"), - "dynamic compose artifact should rewrite sidecar scenario scope onto the live run scope", - ); - assert!( - !child_compose_yaml.contains("AMBER_SCENARIO_SCOPE=scope"), - "dynamic compose artifact should not retain the compiled child scope", - ); - let mut child_network_document = - read_compose_document(&child_artifact.join("compose.yaml")).expect("compose file"); - let child_networks = compose_networks_mut( - &mut child_network_document, - &child_artifact.join("compose.yaml"), - ) - .expect("compose networks") - .expect("compose networks should exist"); - assert_eq!( - compose_network_subnet( - child_networks - .get(yaml_string("amber_egress_c8-root-net")) - .expect("child egress network should remain present"), - ), - Some(initial_subnet.as_str()), - "dynamic compose filtering should preserve the assigned egress subnet", - ); - - let metadata = load_dynamic_compose_child_metadata(&child_artifact) - .expect("dynamic compose child metadata should be readable"); - assert_eq!( - metadata.services, - vec!["c8-root".to_string(), "c8-root-net".to_string()] - ); - assert_eq!( - metadata.readiness_services, - vec!["c8-root".to_string(), "c8-root-net".to_string()] - ); - - let filtered_plan: MeshProvisionPlan = read_json( - &child_artifact.join("mesh-provision-plan.json"), - "mesh provision plan", - ) - .expect("filtered mesh provision plan should exist"); - assert_eq!(filtered_plan.targets.len(), 1); - let component = filtered_plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - .expect("filtered component target should exist"); - let MeshProvisionOutput::Filesystem { dir } = &component.output else { - panic!("filtered component mesh output should be filesystem"); - }; - assert_eq!(dir, ".amber/mesh/c8-root-net"); - assert_eq!( - component - .config - .inbound - .iter() - .map(|route| route.route_id.as_str()) - .collect::>(), - vec!["component:/job/root:http:http"] - ); - assert_eq!( - component.config.inbound[0].allowed_issuers, - vec!["/site/compose_local/router".to_string()] - ); - - let mesh_config: MeshConfigPublic = read_json( - &child_artifact - .join(".amber/mesh/c8-root-net") - .join("mesh-config.json"), - "mesh config", - ) - .expect("child sidecar mesh config should exist"); - let router_peer = mesh_config - .peers - .iter() - .find(|peer| peer.id == "/site/compose_local/router") - .expect("child sidecar should peer with the site router"); - assert_eq!(router_peer.public_key, [7u8; 32]); - - let overlay: StoredRouteOverlayPayload = read_json( - &dynamic_route_overlay_path(&child_artifact), - "site router overlay", - ) - .expect("compose child router overlay should exist"); - assert_eq!( - overlay - .peers - .iter() - .map(|peer| peer.id.as_str()) - .collect::>(), - vec!["/job/root"] - ); - assert_eq!( - overlay - .inbound_routes - .iter() - .map(|route| route.route_id.clone()) - .collect::>(), - vec![ - router_export_route_id("http", MeshProtocol::Http), - "cap_child".to_string(), - ] - ); -} - -#[test] -fn prepare_dynamic_compose_child_artifact_rewrites_same_site_static_provider_inputs() { - let temp = tempdir().expect("tempdir should be created"); - let site_artifact = temp.path().join("site-artifact"); - let child_artifact = temp.path().join("child-artifact"); - fs::create_dir_all(&site_artifact).expect("site artifact dir should exist"); - fs::create_dir_all(&child_artifact).expect("child artifact dir should exist"); - - let mesh_plan = MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![ - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/provider".to_string(), - mesh_scope: Some("compiled-scope".to_string()), - }, - mesh_listen: SocketAddr::from(([0, 0, 0, 0], 23000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/compose_local/router".to_string(), - }], - inbound: vec![InboundRoute { - route_id: component_route_id("/provider", "http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 8080 }, - allowed_issuers: vec!["/site/compose_local/router".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "/amber/provision/c4-provider-net".to_string(), - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/sibling".to_string(), - mesh_scope: Some("compiled-scope".to_string()), - }, - mesh_listen: SocketAddr::from(([0, 0, 0, 0], 23000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/provider".to_string(), - }], - inbound: Vec::new(), - outbound: vec![OutboundRoute { - route_id: component_route_id("/provider", "http", MeshProtocol::Http), - rewrite_route_id: None, - slot: "upstream".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - listen_port: 20000, - listen_addr: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - peer_addr: "c4-provider-net:23000".to_string(), - peer_id: "/provider".to_string(), - capability: "http".to_string(), - }], - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "/amber/provision/c5-sibling-net".to_string(), - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Router, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/site/compose_local/router".to_string(), - mesh_scope: Some("compiled-scope".to_string()), - }, - mesh_listen: SocketAddr::from(([0, 0, 0, 0], 24000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/provider".to_string(), - }], - inbound: vec![InboundRoute { - route_id: router_export_route_id("provider_http", MeshProtocol::Http), - capability: "provider_http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::MeshForward { - peer_addr: "c4-provider-net:23000".to_string(), - peer_id: "/provider".to_string(), - route_id: component_route_id("/provider", "http", MeshProtocol::Http), - capability: "http".to_string(), - }, - allowed_issuers: vec!["/site/compose_local/router".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "/amber/provision/amber-router".to_string(), - }, - }, - ], - }; - let mesh_plan_json = serde_json::to_string(&mesh_plan).expect("mesh plan should serialize"); - fs::write( - child_artifact.join("compose.yaml"), - format!( - r#" -services: - amber-router: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - amber-otelcol: - image: otel/opentelemetry-collector-contrib:0.143.0 - amber-router-control-init: - image: busybox:1.36.1 - amber-provisioner: - image: ghcr.io/rdi-foundation/amber-provisioner:v0.1 - c4-provider: - image: python:3.13-alpine - depends_on: - - c4-provider-net - c4-provider-net: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - volumes: - - c4-provider-net-mesh:/amber/mesh:ro - c5-sibling: - image: python:3.13-alpine - depends_on: - - c5-sibling-net - - c4-provider - c5-sibling-net: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - depends_on: - - amber-provisioner - environment: - - AMBER_SCENARIO_SCOPE=compiled-scope - volumes: - - c5-sibling-net-mesh:/amber/mesh:ro -configs: - amber-mesh-provision-plan: - content: '{mesh_plan_json}' -volumes: - c4-provider-net-mesh: {{}} - c5-sibling-net-mesh: {{}} -networks: - amber_mesh: - driver: bridge - amber_boundary: - driver: bridge -x-amber: - version: "1" - router: - mesh_port: 24000 - control_port: 24100 -"# - ), - ) - .expect("dynamic compose artifact should be written"); - - fs::write( - site_artifact.join("compose.yaml"), - r#" -services: - amber-router: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - amber-otelcol: - image: otel/opentelemetry-collector-contrib:0.143.0 - amber-router-control-init: - image: busybox:1.36.1 - c4-provider: - image: python:3.13-alpine - depends_on: - - c4-provider-net - c4-provider-net: - image: ghcr.io/rdi-foundation/amber-router:v0.1 - volumes: - - c4-provider-net-mesh:/amber/mesh:ro -volumes: - c4-provider-net-mesh: {} -x-amber: - version: "1" - router: - mesh_port: 24000 - control_port: 24100 -"#, - ) - .expect("site compose artifact should be written"); - - prepare_dynamic_compose_child_artifact( - &SiteControllerRuntimePlan { - schema: SITE_PLAN_SCHEMA.to_string(), - version: SITE_PLAN_VERSION, - run_id: "run-123".to_string(), - mesh_scope: "live-scope".to_string(), - run_root: temp.path().display().to_string(), - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - router_identity_id: "/site/compose_local/router".to_string(), - artifact_dir: site_artifact.display().to_string(), - site_state_root: temp.path().display().to_string(), - listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: Some("amber-test".to_string()), - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), - }, - &DynamicSitePlanRecord { - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - router_identity_id: "/site/compose_local/router".to_string(), - component_ids: vec![5], - assigned_components: vec!["/sibling".to_string()], - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: vec![DynamicInputRouteRecord { - component: "/sibling".to_string(), - slot: "upstream".to_string(), - provider_component: "/provider".to_string(), - protocol: "http".to_string(), - capability_kind: "http".to_string(), - capability_profile: None, - target: DynamicInputRouteTarget::ComponentProvide { - provide: "http".to_string(), - }, - }], - }, - &child_artifact, - &[], - &BTreeMap::from([ - ( - "/site/compose_local/router".to_string(), - MeshIdentityPublic { - id: "/site/compose_local/router".to_string(), - public_key: [7; 32], - mesh_scope: Some("live-scope".to_string()), - }, - ), - ( - "/provider".to_string(), - MeshIdentityPublic { - id: "/provider".to_string(), - public_key: [8; 32], - mesh_scope: Some("live-scope".to_string()), - }, - ), - ]), - ) - .expect("dynamic compose child artifact should be prepared"); - - let filtered_plan: MeshProvisionPlan = read_json( - &child_artifact.join("mesh-provision-plan.json"), - "mesh provision plan", - ) - .expect("filtered mesh provision plan should exist"); - assert_eq!(filtered_plan.targets.len(), 1); - let component = filtered_plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - .expect("filtered component target should exist"); - assert_eq!(component.config.identity.id, "/sibling"); - assert!( - component - .config - .peers - .iter() - .any(|peer| peer.id == "/site/compose_local/router"), - "rewritten child mesh plan should peer with the site router", - ); - assert_eq!(component.config.outbound.len(), 1); - let outbound = &component.config.outbound[0]; - assert_eq!(outbound.slot, "upstream"); - assert_eq!( - outbound.route_id, - component_route_id("/provider", "http", MeshProtocol::Http) - ); - assert_eq!( - outbound.peer_addr, - format!("{COMPOSE_ROUTER_SERVICE_NAME}:24000") - ); - assert_eq!(outbound.peer_id, "/site/compose_local/router"); - assert_eq!(outbound.capability, "http"); - - let embedded_plan = read_embedded_compose_mesh_provision_plan(&child_artifact) - .expect("embedded compose mesh plan should be readable"); - let embedded_component = embedded_plan - .targets - .iter() - .find(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && target.config.identity.id == "/sibling" - }) - .expect("embedded child component target should exist"); - assert_eq!(embedded_component.config.outbound.len(), 1); - assert_eq!( - embedded_component.config.outbound[0].peer_addr, - format!("{COMPOSE_ROUTER_SERVICE_NAME}:24000") - ); - assert_eq!( - embedded_component.config.outbound[0].peer_id, - "/site/compose_local/router" - ); - - let overlay: StoredRouteOverlayPayload = read_json( - &dynamic_route_overlay_path(&child_artifact), - "site router overlay", - ) - .expect("compose child router overlay should exist"); - assert!( - overlay.peers.iter().any(|peer| peer.id == "/provider"), - "compose routed-input overlay should include the provider peer", - ); - let routed_input_route = overlay - .inbound_routes - .iter() - .find(|route| route.route_id == component_route_id("/provider", "http", MeshProtocol::Http)) - .expect("compose routed-input overlay should include the provider route"); - let InboundTarget::MeshForward { - peer_addr, - peer_id, - route_id, - capability, - } = &routed_input_route.target - else { - panic!("compose routed-input overlay should forward across mesh"); - }; - assert_eq!(peer_addr, "c4-provider-net:23000"); - assert_eq!(peer_id, "/provider"); - assert_eq!(route_id, "component:/provider:http:http"); - assert_eq!(capability, "http"); -} - -#[test] -fn dynamic_route_issuer_grants_include_component_provide_inputs() { - let issuers = dynamic_route_issuer_grants(&[SiteControllerRuntimeChildRecord { - child_id: 7, - artifact_root: "/tmp/child".to_string(), - assigned_components: vec!["/sibling".to_string()], - proxy_exports: BTreeMap::new(), - routed_inputs: vec![DynamicInputRouteRecord { - component: "/sibling".to_string(), - slot: "upstream".to_string(), - provider_component: "/provider".to_string(), - protocol: "http".to_string(), - capability_kind: "http".to_string(), - capability_profile: None, - target: DynamicInputRouteTarget::ComponentProvide { - provide: "http".to_string(), - }, - }], - process_pid: None, - published: true, - }]) - .expect("component-provide routed inputs should produce issuer grants"); - - assert_eq!( - issuers.get("component:/provider:http:http"), - Some(&BTreeSet::from(["/sibling".to_string()])) - ); -} - -#[test] -fn direct_startup_route_overlay_does_not_require_materialized_child_mesh_files() { - let temp = tempdir().expect("tempdir should be created"); - let artifact_root = temp.path().join("child-artifact"); - fs::create_dir_all(&artifact_root).expect("child artifact dir should exist"); - write_json( - &artifact_root.join("mesh-provision-plan.json"), - &MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![ - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/consumer".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 0)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/direct_local/router".to_string(), - }], - inbound: Vec::new(), - outbound: vec![OutboundRoute { - route_id: "placeholder".to_string(), - rewrite_route_id: None, - slot: "upstream".to_string(), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - listen_port: 21000, - listen_addr: None, - peer_addr: "127.0.0.1:1".to_string(), - peer_id: "/provider".to_string(), - http_plugins: Vec::new(), - }], - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/components/c1-consumer".to_string(), - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Router, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/site/direct_local/router".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 0)), - control_listen: Some(SocketAddr::from(([127, 0, 0, 1], 0))), - dynamic_caps_listen: None, - control_allow: None, - peers: Vec::new(), - inbound: Vec::new(), - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/router".to_string(), - }, - }, - ], - }, - ) - .expect("mesh provision plan should be written"); - - write_direct_vm_startup_route_overlay_payload( - &artifact_root, - "direct", - &[DynamicInputRouteRecord { - component: "/consumer".to_string(), - slot: "upstream".to_string(), - provider_component: "/provider".to_string(), - protocol: "http".to_string(), - capability_kind: "http".to_string(), - capability_profile: None, - target: DynamicInputRouteTarget::ComponentProvide { - provide: "http".to_string(), - }, - }], - &BTreeMap::from([("/provider".to_string(), "127.0.0.1:25000".to_string())]), - &BTreeMap::from([( - "/provider".to_string(), - MeshIdentityPublic { - id: "/provider".to_string(), - public_key: [9; 32], - mesh_scope: Some("scope".to_string()), - }, - )]), - ) - .expect("startup overlay should be written without child mesh files in the artifact"); - - let overlay: StoredRouteOverlayPayload = read_json( - &dynamic_route_overlay_path(&artifact_root), - "site router overlay", - ) - .expect("startup overlay should exist"); - assert_eq!(overlay.peers.len(), 1); - assert_eq!(overlay.peers[0].id, "/provider"); - let route = overlay - .inbound_routes - .iter() - .find(|route| route.route_id == component_route_id("/provider", "http", MeshProtocol::Http)) - .expect("startup overlay should include the provider route"); - let InboundTarget::MeshForward { - peer_addr, - peer_id, - route_id, - capability, - } = &route.target - else { - panic!("startup overlay should forward routed inputs over mesh"); - }; - assert_eq!(peer_addr, "127.0.0.1:25000"); - assert_eq!(peer_id, "/provider"); - assert_eq!(route_id, "component:/provider:http:http"); - assert_eq!(capability, "http"); -} - -#[test] -fn filter_dynamic_mesh_provision_plan_keeps_only_child_owned_router_routes() { - let temp = tempdir().expect("tempdir should be created"); - let artifact_root = temp.path(); - write_json( - &artifact_root.join("mesh-provision-plan.json"), - &MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![ - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/vm_admin".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 0)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/vm_local/router".to_string(), - }], - inbound: vec![InboundRoute { - route_id: component_route_id("/vm_admin", "http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 8080 }, - allowed_issuers: vec!["/site/vm_local/router".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/components/c4-vm_admin".to_string(), - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/job-compose/vm_helper/root".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 0)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/vm_local/router".to_string(), - }], - inbound: vec![InboundRoute { - route_id: component_route_id( - "/job-compose/vm_helper/root", - "http", - MeshProtocol::Http, - ), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 8080 }, - allowed_issuers: vec!["/site/vm_local/router".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/components/c9-root".to_string(), - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Router, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/site/vm_local/router".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 0)), - control_listen: Some(SocketAddr::from(([127, 0, 0, 1], 0))), - dynamic_caps_listen: None, - control_allow: None, - peers: vec![ - amber_mesh::MeshPeerTemplate { - id: "/vm_admin".to_string(), - }, - amber_mesh::MeshPeerTemplate { - id: "/job-compose/vm_helper/root".to_string(), - }, - ], - inbound: vec![ - InboundRoute { - route_id: router_export_route_id( - "vm_admin_http", - MeshProtocol::Http, - ), - capability: "vm_admin_http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::MeshForward { - peer_addr: "127.0.0.1:23001".to_string(), - peer_id: "/vm_admin".to_string(), - route_id: component_route_id( - "/vm_admin", - "http", - MeshProtocol::Http, - ), - capability: "http".to_string(), - }, - allowed_issuers: vec!["/site/vm_local/router".to_string()], - }, - InboundRoute { - route_id: router_export_route_id( - "amber_export_916351bacd5bad90", - MeshProtocol::Http, - ), - capability: "amber_export_916351bacd5bad90".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::MeshForward { - peer_addr: "127.0.0.1:23002".to_string(), - peer_id: "/job-compose/vm_helper/root".to_string(), - route_id: component_route_id( - "/job-compose/vm_helper/root", - "http", - MeshProtocol::Http, - ), - capability: "http".to_string(), - }, - allowed_issuers: vec!["/site/vm_local/router".to_string()], - }, - InboundRoute { - route_id: "cap_static_admin".to_string(), - capability: "component".to_string(), - capability_kind: Some("component".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL".to_string(), - optional: false, - }, - allowed_issuers: vec!["/vm_admin".to_string()], - }, - InboundRoute { - route_id: "cap_child".to_string(), - capability: "component".to_string(), - capability_kind: Some("component".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL".to_string(), - optional: false, - }, - allowed_issuers: vec!["/job-compose/vm_helper/root".to_string()], - }, - ], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/router".to_string(), - }, - }, - ], - }, - ) - .expect("mesh provision plan should be written"); - - filter_dynamic_mesh_provision_plan( - artifact_root, - &BTreeSet::from(["mesh/components/c9-root".to_string()]), - ) - .expect("dynamic mesh provision plan should be filtered"); - - let filtered: MeshProvisionPlan = read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - ) - .expect("filtered mesh provision plan should be readable"); - assert_eq!(filtered.targets.len(), 2); - - let component = filtered - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - .expect("filtered child component target should remain"); - assert_eq!(component.config.identity.id, "/job-compose/vm_helper/root"); - - let router = filtered - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - .expect("filtered router target should remain"); - assert_eq!( - router - .config - .peers - .iter() - .map(|peer| peer.id.as_str()) - .collect::>(), - vec!["/job-compose/vm_helper/root"] - ); - assert_eq!( - router - .config - .inbound - .iter() - .map(|route| route.route_id.clone()) - .collect::>(), - vec![ - router_export_route_id("amber_export_916351bacd5bad90", MeshProtocol::Http), - "cap_child".to_string(), - ] - ); -} - -#[test] -fn ensure_dynamic_proxy_export_component_routes_in_artifact_adds_provider_route() { - let temp = tempdir().expect("tempdir should be created"); - let artifact_root = temp.path(); - write_json( - &artifact_root.join("mesh-provision-plan.json"), - &MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/job/root".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 23000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: Vec::new(), - inbound: vec![InboundRoute { - route_id: component_route_id("/job/root", "admin", MeshProtocol::Http), - capability: "admin".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 9090 }, - allowed_issuers: Vec::new(), - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/components/c8-root".to_string(), - }, - }], - }, - ) - .expect("mesh provision plan should be written"); - - ensure_dynamic_proxy_export_component_routes_in_artifact( - artifact_root, - &BTreeMap::from([( - "http".to_string(), - test_dynamic_proxy_export_record(8, "/job/root", "http", "http", "http", 8080), - )]), - "/site/direct_local/router", - ) - .expect("dynamic provider routes should be projected into the child mesh plan"); - - let filtered: MeshProvisionPlan = read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - ) - .expect("mesh provision plan should be readable"); - let component = filtered - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - .expect("component target should remain"); - assert_eq!( - component - .config - .peers - .iter() - .map(|peer| peer.id.as_str()) - .collect::>(), - vec!["/site/direct_local/router"] - ); - let admin_route = component - .config - .inbound - .iter() - .find(|route| route.capability == "admin") - .expect("existing local admin route should remain"); - assert!( - admin_route.allowed_issuers.is_empty(), - "rewriting a dynamic export must not broaden unrelated local routes", - ); - let provider_route = component - .config - .inbound - .iter() - .find(|route| route.capability == "http") - .expect("dynamic export provider route should be added"); - assert_eq!( - provider_route.allowed_issuers, - vec!["/site/direct_local/router".to_string()] - ); -} - #[test] -fn ensure_dynamic_proxy_export_component_routes_in_artifact_leaves_local_routes_closed_without_exports() - { - let temp = tempdir().expect("tempdir should be created"); - let artifact_root = temp.path(); - write_json( - &artifact_root.join("mesh-provision-plan.json"), - &MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/job/root".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 23000)), - control_listen: None, - dynamic_caps_listen: Some(SocketAddr::from(([127, 0, 0, 1], 19000))), - control_allow: None, - peers: Vec::new(), - inbound: vec![ - InboundRoute { - route_id: component_route_id("/job/root", "http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 8080 }, - allowed_issuers: Vec::new(), - }, - InboundRoute { - route_id: "external".to_string(), - capability: "external".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "UPSTREAM".to_string(), - optional: false, - }, - allowed_issuers: Vec::new(), - }, - ], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/components/c8-root".to_string(), - }, - }], - }, - ) - .expect("mesh provision plan should be written"); - - ensure_dynamic_proxy_export_component_routes_in_artifact( - artifact_root, - &BTreeMap::new(), - "/site/direct_local/router", - ) - .expect("empty dynamic exports should leave existing routes unchanged"); - - let filtered: MeshProvisionPlan = read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - ) - .expect("mesh provision plan should be readable"); - let component = filtered - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - .expect("component target should remain"); - assert!( - component.config.peers.is_empty(), - "the router should not be peered when there are no dynamic exports to expose", - ); +fn site_state_paths_are_site_scoped() { + let root = Path::new("/tmp/amber-run/state"); assert_eq!( - component.config.inbound[0].allowed_issuers, - Vec::::new() - ); - assert!( - component.config.inbound[1].allowed_issuers.is_empty(), - "non-local routes should also remain unchanged" + site_state_path(root, "site-a"), + Path::new("/tmp/amber-run/state/site-a/manager-state.json") ); -} - -#[test] -fn compose_component_mesh_peer_addr_uses_service_name() { - let temp = tempdir().expect("tempdir should be created"); - let addr = compose_component_mesh_peer_addr( - temp.path(), - "/provider", - &MeshProvisionOutput::Filesystem { - dir: "mesh/components/c4-provider-net".to_string(), - }, - 23000, - ) - .expect("compose peer addr should resolve"); - assert_eq!(addr, "c4-provider-net:23000"); -} - -#[test] -fn kubernetes_component_mesh_peer_addr_uses_service_name() { - let temp = tempdir().expect("tempdir should be created"); - let addr = kubernetes_component_mesh_peer_addr( - temp.path(), - "/provider", - &MeshProvisionOutput::KubernetesSecret { - name: "c4-provider-net-mesh".to_string(), - namespace: Some("ns".to_string()), - }, - 23000, - ) - .expect("kubernetes peer addr should resolve"); - assert_eq!(addr, "c4-provider-net:23000"); -} - -#[test] -fn add_dynamic_proxy_export_overlay_routes_rewrites_existing_export_route() { - let mut inbound_routes = vec![InboundRoute { - route_id: router_dynamic_export_route_id("/job/root", "http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::MeshForward { - peer_addr: "stale-service:23000".to_string(), - peer_id: "/job/root".to_string(), - route_id: component_route_id("/job/root", "http", MeshProtocol::Http), - capability: "http".to_string(), - }, - allowed_issuers: vec!["/site/kind_local/router".to_string()], - }]; - - add_dynamic_proxy_export_overlay_routes( - &mut inbound_routes, - &BTreeMap::from([( - "http".to_string(), - test_dynamic_proxy_export_record(8, "/job/root", "http", "http", "http", 8080), - )]), - |_| Ok("current-service:23000".to_string()), - ) - .expect("dynamic export routes should be rewritten from the current artifact state"); - - assert_eq!(inbound_routes.len(), 1); - let InboundTarget::MeshForward { - peer_addr, - peer_id, - route_id, - capability, - } = &inbound_routes[0].target - else { - panic!("export route should remain a mesh forward"); - }; - assert_eq!(peer_addr, "current-service:23000"); - assert_eq!(peer_id, "/job/root"); - assert_eq!(route_id, "component:/job/root:http:http"); - assert_eq!(capability, "http"); -} - -#[test] -fn child_router_overlay_payload_synthesizes_dynamic_proxy_export_routes_for_direct_children() { - let temp = tempdir().expect("tempdir should be created"); - let artifact_root = temp.path().join("artifact"); - let runtime_root = temp.path().join("runtime"); - fs::create_dir_all(&artifact_root).expect("artifact root should be created"); - fs::create_dir_all(runtime_root.join("mesh/components/c8-root")) - .expect("runtime mesh dir should be created"); - - write_json( - &artifact_root.join("mesh-provision-plan.json"), - &MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![ - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/job/root".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 23000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/direct_local/router".to_string(), - }], - inbound: vec![InboundRoute { - route_id: "cap_child".to_string(), - capability: "component".to_string(), - capability_kind: Some("component".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL".to_string(), - optional: false, - }, - allowed_issuers: vec!["/job/root".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/components/c8-root".to_string(), - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Router, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/site/direct_local/router".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 24000)), - control_listen: Some(SocketAddr::from(([127, 0, 0, 1], 24100))), - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/job/root".to_string(), - }], - inbound: vec![InboundRoute { - route_id: "cap_child".to_string(), - capability: "component".to_string(), - capability_kind: Some("component".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL".to_string(), - optional: false, - }, - allowed_issuers: vec!["/job/root".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::Filesystem { - dir: "mesh/router".to_string(), - }, - }, - ], - }, - ) - .expect("mesh provision plan should be written"); - write_dynamic_proxy_exports_metadata( - &artifact_root, - &BTreeMap::from([( - "http".to_string(), - test_dynamic_proxy_export_record(8, "/job/root", "http", "http", "http", 8080), - )]), - ) - .expect("dynamic proxy export metadata should be written"); - let identity = MeshIdentity::generate("/job/root", Some("scope".to_string())); - write_json( - &runtime_root - .join("mesh/components/c8-root") - .join(MESH_IDENTITY_FILENAME), - &MeshIdentitySecret::from_identity(&identity), - ) - .expect("mesh identity should be written"); - write_json( - &runtime_root - .join("mesh/components/c8-root") - .join(MESH_CONFIG_FILENAME), - &MeshConfigPublic { - identity: MeshIdentityPublic { - id: "/job/root".to_string(), - public_key: identity.public_key, - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 26000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: Vec::new(), - inbound: Vec::new(), - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - ) - .expect("mesh config should be written"); - - let (_peers, inbound_routes) = child_router_overlay_payload( - &SiteControllerRuntimePlan { - schema: SITE_CONTROLLER_RUNTIME_PLAN_SCHEMA.to_string(), - version: SITE_CONTROLLER_RUNTIME_PLAN_VERSION, - run_id: "run-123".to_string(), - mesh_scope: "mesh-scope-test".to_string(), - run_root: temp.path().display().to_string(), - site_id: "direct_local".to_string(), - kind: SiteKind::Direct, - router_identity_id: "/site/direct_local/router".to_string(), - artifact_dir: artifact_root.display().to_string(), - site_state_root: temp.path().join("state").display().to_string(), - listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), - storage_root: Some(temp.path().join("storage").display().to_string()), - runtime_root: Some(runtime_root.display().to_string()), - router_mesh_port: Some(24000), - compose_project: None, - kubernetes_namespace: None, - context: None, - observability_endpoint: None, - launch_env: BTreeMap::new(), - }, - &artifact_root, - &runtime_root, - ) - .expect("child router overlay payload should be synthesized"); - - let export_route = inbound_routes - .iter() - .find(|route| { - route.route_id - == router_dynamic_export_route_id("/job/root", "http", MeshProtocol::Http) - }) - .expect("dynamic export route should be synthesized"); - let InboundTarget::MeshForward { - peer_addr, - peer_id, - route_id, - capability, - } = &export_route.target - else { - panic!("dynamic export route should forward to the child provider"); - }; - assert_eq!(peer_addr, "127.0.0.1:26000"); - assert_eq!(peer_id, "/job/root"); - assert_eq!(route_id, "component:/job/root:http:http"); - assert_eq!(capability, "http"); -} - -#[test] -fn prepare_dynamic_kubernetes_child_artifact_keeps_router_overlay_local() { - let temp = tempdir().expect("tempdir should be created"); - let configmaps_dir = temp.path().join("01-configmaps"); - let rbac_dir = temp.path().join("02-rbac"); - let deployments_dir = temp.path().join("03-deployments"); - let services_dir = temp.path().join("04-services"); - let netpol_dir = temp.path().join("05-networkpolicies"); - let site_state_root = temp.path().join("state").join("kind_local"); - fs::create_dir_all(&configmaps_dir).expect("configmaps dir should exist"); - fs::create_dir_all(&rbac_dir).expect("rbac dir should exist"); - fs::create_dir_all(&deployments_dir).expect("deployments dir should exist"); - fs::create_dir_all(&services_dir).expect("services dir should exist"); - fs::create_dir_all(&netpol_dir).expect("network policies dir should exist"); - fs::create_dir_all(&site_state_root).expect("site state dir should exist"); - - let full_plan = MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![ - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/kind_admin".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 0)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/kind_local/router".to_string(), - }], - inbound: vec![InboundRoute { - route_id: component_route_id("/kind_admin", "http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 8080 }, - allowed_issuers: vec!["/site/kind_local/router".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::KubernetesSecret { - name: "c3-kind-admin-mesh".to_string(), - namespace: None, - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/job/root".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 23007)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: vec![amber_mesh::MeshPeerTemplate { - id: "/site/kind_local/router".to_string(), - }], - inbound: vec![InboundRoute { - route_id: component_route_id("/job/root", "http", MeshProtocol::Http), - capability: "http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::Local { port: 8080 }, - allowed_issuers: vec!["/site/kind_local/router".to_string()], - }], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::KubernetesSecret { - name: "c7-kind-helper-mesh".to_string(), - namespace: None, - }, - }, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Router, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/site/kind_local/router".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 0)), - control_listen: Some(SocketAddr::from(([127, 0, 0, 1], 0))), - dynamic_caps_listen: None, - control_allow: None, - peers: vec![ - amber_mesh::MeshPeerTemplate { - id: "/kind_admin".to_string(), - }, - amber_mesh::MeshPeerTemplate { - id: "/job/root".to_string(), - }, - ], - inbound: vec![ - InboundRoute { - route_id: router_export_route_id("kind_http", MeshProtocol::Http), - capability: "kind_http".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::MeshForward { - peer_addr: "c3-kind-admin:23000".to_string(), - peer_id: "/kind_admin".to_string(), - route_id: component_route_id( - "/kind_admin", - "http", - MeshProtocol::Http, - ), - capability: "http".to_string(), - }, - allowed_issuers: vec!["/site/kind_local/router".to_string()], - }, - InboundRoute { - route_id: router_export_route_id( - "amber_export_5062ceb53a8ac5c2", - MeshProtocol::Http, - ), - capability: "amber_export_5062ceb53a8ac5c2".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::MeshForward { - peer_addr: "c7-kind-helper:23000".to_string(), - peer_id: "/job/root".to_string(), - route_id: component_route_id( - "/job/root", - "http", - MeshProtocol::Http, - ), - capability: "http".to_string(), - }, - allowed_issuers: vec!["/site/kind_local/router".to_string()], - }, - InboundRoute { - route_id: "cap_static_admin".to_string(), - capability: "component".to_string(), - capability_kind: Some("component".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL".to_string(), - optional: false, - }, - allowed_issuers: vec!["/kind_admin".to_string()], - }, - InboundRoute { - route_id: "cap_child".to_string(), - capability: "component".to_string(), - capability_kind: Some("component".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL".to_string(), - optional: false, - }, - allowed_issuers: vec!["/job/root".to_string()], - }, - ], - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::KubernetesSecret { - name: "amber-router-external".to_string(), - namespace: None, - }, - }, - ], - }; - - let full_plan_json = - serde_json::to_string_pretty(&full_plan).expect("mesh plan should serialize"); - let indented_json = full_plan_json - .lines() - .map(|line| format!(" {line}")) - .collect::>() - .join("\n"); - fs::write( - configmaps_dir.join("amber-mesh-provision.yaml"), - format!( - "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: amber-mesh-provision\ndata:\n \ - mesh-plan.json: |-\n{indented_json}\n" - ), - ) - .expect("mesh provision configmap should be written"); - for (path, body) in - [ - ( - temp.path().join("kustomization.yaml"), - r#"apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- 01-configmaps/amber-mesh-provision.yaml -- 02-rbac/amber-provisioner-job.yaml -- 02-rbac/amber-provisioner-role.yaml -- 02-rbac/amber-provisioner-rolebinding.yaml -- 02-rbac/amber-provisioner-sa.yaml -- 03-deployments/amber-router.yaml -- 03-deployments/c3-kind-admin.yaml -- 03-deployments/c7-kind-helper.yaml -- 04-services/amber-router.yaml -- 04-services/c3-kind-admin.yaml -- 04-services/c7-kind-helper.yaml -- 05-networkpolicies/amber-router-netpol.yaml -- 05-networkpolicies/c3-kind-admin-netpol.yaml -- 05-networkpolicies/c7-kind-helper-netpol.yaml -secretGenerator: -- name: amber-router-external - envs: - - router-external.env -"#, - ), - ( - temp.path().join("router-external.env"), - "AMBER_EXTERNAL_SLOT_API_URL=\n", - ), - ( - rbac_dir.join("amber-provisioner-job.yaml"), - "apiVersion: batch/v1\nkind: Job\nmetadata:\n name: amber-provisioner-bdcf48c3\n", - ), - ( - rbac_dir.join("amber-provisioner-role.yaml"), - "apiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n name: \ - amber-provisioner\n", - ), - ( - rbac_dir.join("amber-provisioner-rolebinding.yaml"), - "apiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: \ - amber-provisioner\n", - ), - ( - rbac_dir.join("amber-provisioner-sa.yaml"), - "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: amber-provisioner\n", - ), - ( - deployments_dir.join("amber-router.yaml"), - "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: amber-router\n \ - labels:\n amber.io/component-id: router\n", - ), - ( - deployments_dir.join("c3-kind-admin.yaml"), - "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: c3-kind-admin\n \ - labels:\n amber.io/component-id: c3\n", - ), - ( - deployments_dir.join("c7-kind-helper.yaml"), - "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: c7-kind-helper\n \ - labels:\n amber.io/component-id: c7\nspec:\n template:\n spec:\n \ - initContainers:\n - name: wait-mesh-config\n command:\n - \ - /amber-helper\n - wait-mesh-config\n - \ - /amber/mesh/mesh-config.json\n - scope\n - '30'\n \ - containers:\n - name: sidecar\n env:\n - name: \ - AMBER_SCENARIO_SCOPE\n value: scope\n", - ), - ( - services_dir.join("amber-router.yaml"), - "apiVersion: v1\nkind: Service\nmetadata:\n name: amber-router\n labels:\n \ - amber.io/component-id: router\n", - ), - ( - services_dir.join("c3-kind-admin.yaml"), - "apiVersion: v1\nkind: Service\nmetadata:\n name: c3-kind-admin\n labels:\n \ - amber.io/component-id: c3\n", - ), - ( - services_dir.join("c7-kind-helper.yaml"), - "apiVersion: v1\nkind: Service\nmetadata:\n name: c7-kind-helper\n labels:\n \ - amber.io/component-id: c7\n", - ), - ( - netpol_dir.join("amber-router-netpol.yaml"), - "apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\n name: \ - amber-router-netpol\n labels:\n amber.io/component-id: router\n", - ), - ( - netpol_dir.join("c3-kind-admin-netpol.yaml"), - "apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\n name: \ - c3-kind-admin-netpol\n labels:\n amber.io/component-id: c3\n", - ), - ( - netpol_dir.join("c7-kind-helper-netpol.yaml"), - "apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\n name: \ - c7-kind-helper-netpol\n labels:\n amber.io/component-id: c7\nspec:\n \ - podSelector:\n matchLabels:\n amber.io/component: c7-kind-helper\n \ - policyTypes:\n - Ingress\n ingress:\n - from:\n - podSelector:\n \ - matchLabels:\n amber.io/component: c3-kind-admin\n ports:\n - \ - protocol: TCP\n port: 23007\n", - ), - ] - { - fs::write(path, body).expect("kubernetes artifact file should be written"); - } - write_json( - &site_state_root.join("manager-state.json"), - &SiteManagerState { - schema: SITE_STATE_SCHEMA.to_string(), - version: SITE_STATE_VERSION, - run_id: "run-123".to_string(), - site_id: "kind_local".to_string(), - kind: SiteKind::Kubernetes, - status: SiteLifecycleStatus::Running, - artifact_dir: temp.path().display().to_string(), - supervisor_pid: 1, - process_pid: None, - compose_project: None, - kubernetes_namespace: Some("amber-run-123-kind-local".to_string()), - port_forward_pid: None, - context: Some("kind-test".to_string()), - router_control: Some("127.0.0.1:24100".to_string()), - router_mesh_addr: Some("127.0.0.1:24000".to_string()), - router_identity_id: Some("/site/kind_local/router".to_string()), - router_public_key_b64: Some( - base64::engine::general_purpose::STANDARD.encode([7u8; 32]), - ), - site_controller_pid: None, - site_controller_url: None, - last_error: None, - }, - ) - .expect("site manager state should be written"); - - prepare_dynamic_kubernetes_child_artifact( - &SiteControllerRuntimePlan { - schema: SITE_CONTROLLER_RUNTIME_PLAN_SCHEMA.to_string(), - version: SITE_PLAN_VERSION, - run_id: "run-123".to_string(), - mesh_scope: "mesh-scope-test".to_string(), - run_root: temp.path().display().to_string(), - site_id: "kind_local".to_string(), - kind: SiteKind::Kubernetes, - router_identity_id: "/site/kind_local/router".to_string(), - artifact_dir: temp.path().display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: None, - kubernetes_namespace: Some("amber-run-123-kind-local".to_string()), - context: Some("kind-test".to_string()), - observability_endpoint: None, - launch_env: BTreeMap::new(), - }, - &DynamicSitePlanRecord { - site_id: "kind_local".to_string(), - kind: SiteKind::Kubernetes, - router_identity_id: "/site/kind_local/router".to_string(), - component_ids: vec![7], - assigned_components: vec!["/job/root".to_string()], - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::from([( - "http".to_string(), - test_dynamic_proxy_export_record(7, "/job/root", "http", "http", "http", 8080), - )]), - routed_inputs: Vec::new(), - }, - temp.path(), - &BTreeMap::from([( - "/site/kind_local/router".to_string(), - MeshIdentityPublic { - id: "/site/kind_local/router".to_string(), - public_key: [7; 32], - mesh_scope: Some("mesh-scope-test".to_string()), - }, - )]), - ) - .expect("kubernetes dynamic child artifact should be prepared"); - - let overlay_plan: MeshProvisionPlan = read_json( - &temp.path().join("mesh-provision-plan.json"), - "mesh provision plan", - ) - .expect("overlay mesh plan should be readable"); - assert_eq!(overlay_plan.targets.len(), 2); - let overlay_scopes = overlay_plan - .targets - .iter() - .map(|target| target.config.identity.mesh_scope.clone()) - .collect::>(); - assert!( - overlay_plan - .targets - .iter() - .all(|target| target.config.identity.mesh_scope.as_deref() == Some("mesh-scope-test")), - "dynamic kubernetes overlay plan should be projected onto the live run mesh scope: \ - {overlay_scopes:?}", - ); - let overlay_router = overlay_plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - .expect("overlay router target should exist"); - assert_eq!( - overlay_router - .config - .peers - .iter() - .map(|peer| peer.id.as_str()) - .collect::>(), - vec!["/job/root"] - ); - assert_eq!( - overlay_router - .config - .inbound - .iter() - .map(|route| route.route_id.clone()) - .collect::>(), - vec![ - router_export_route_id("amber_export_5062ceb53a8ac5c2", MeshProtocol::Http), - "cap_child".to_string(), - ] - ); - - let embedded_plan = read_embedded_kubernetes_mesh_provision_plan(temp.path()) - .expect("embedded kubernetes plan should be readable"); - assert_eq!(embedded_plan.targets.len(), 1); - assert!( - embedded_plan - .targets - .iter() - .all(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) - ); - assert_eq!( - embedded_plan.targets[0].config.identity.id, "/job/root", - "embedded provision plan should keep only child component targets" - ); - assert!( - embedded_plan - .targets - .iter() - .all(|target| target.config.identity.mesh_scope.as_deref() == Some("mesh-scope-test")), - "embedded provision plan should inherit the live run mesh scope", - ); - assert_eq!(embedded_plan.existing_peer_identities.len(), 1); assert_eq!( - embedded_plan.existing_peer_identities[0].id, - "/site/kind_local/router" + desired_links_path(Path::new("/tmp/amber-run/state/site-a")), + Path::new("/tmp/amber-run/state/site-a/desired-links.json") ); assert_eq!( - embedded_plan.existing_peer_identities[0] - .mesh_scope - .as_deref(), - Some("mesh-scope-test"), - "existing router identity should preserve the live run mesh scope", + site_controller_plan_path(Path::new("/tmp/amber-run/state/site-a")), + Path::new("/tmp/amber-run/state/site-a/site-controller-plan.json") ); - let helper_deployment = fs::read_to_string(deployments_dir.join("c7-kind-helper.yaml")) - .expect("kubernetes child deployment should be readable"); - assert!( - helper_deployment.contains("- mesh-scope-test"), - "dynamic kubernetes deployment should rewrite the wait-mesh-config expected scope", - ); - assert!( - helper_deployment.contains("value: mesh-scope-test"), - "dynamic kubernetes deployment should rewrite AMBER_SCENARIO_SCOPE onto the live run scope", - ); - assert!( - !helper_deployment.contains("value: scope"), - "dynamic kubernetes deployment should not retain the compiled child scope", - ); - let helper_netpol = fs::read_to_string(netpol_dir.join("c7-kind-helper-netpol.yaml")) - .expect("kubernetes child network policy should be readable"); - assert!( - helper_netpol.contains("amber.io/component: amber-router"), - "dynamic kubernetes child network policy should admit router ingress for proxy exports", - ); - assert!( - helper_netpol.contains("port: 23007"), - "dynamic kubernetes child network policy should retain the child mesh port for router \ - ingress", - ); - - let kustomization = fs::read_to_string(temp.path().join("kustomization.yaml")) - .expect("kustomization should be readable"); - assert!(kustomization.contains("03-deployments/c7-kind-helper.yaml")); - assert!(kustomization.contains("04-services/c7-kind-helper.yaml")); - assert!(kustomization.contains("05-networkpolicies/c7-kind-helper-netpol.yaml")); - assert!(kustomization.contains("02-rbac/amber-provisioner-job.yaml")); - assert!(!kustomization.contains("03-deployments/amber-router.yaml")); - assert!(!kustomization.contains("03-deployments/c3-kind-admin.yaml")); - assert!(!kustomization.contains("amber-router-external")); - assert!( - !deployments_dir.join("amber-router.yaml").exists(), - "dynamic kubernetes child artifact should delete unrelated router resources", - ); - assert!( - !deployments_dir.join("c3-kind-admin.yaml").exists(), - "dynamic kubernetes child artifact should delete unrelated creator resources", - ); - assert!( - !services_dir.join("amber-router.yaml").exists(), - "dynamic kubernetes child artifact should delete unrelated router services", - ); - assert!( - !services_dir.join("c3-kind-admin.yaml").exists(), - "dynamic kubernetes child artifact should delete unrelated creator services", - ); - - let destroy_bundle = project_kubernetes_dynamic_child_destroy_artifact_files( - &read_artifact_snapshot(temp.path()).expect("prepared child artifact should be readable"), - ) - .expect("destroy bundle should project"); - let destroy_kustomization = destroy_bundle - .get("kustomization.yaml") - .expect("destroy bundle should include kustomization"); - assert!(destroy_kustomization.contains("03-deployments/c7-kind-helper.yaml")); - assert!(destroy_kustomization.contains("02-rbac/amber-provisioner-job.yaml")); - assert!(!destroy_kustomization.contains("01-configmaps/amber-mesh-provision.yaml")); - assert!(!destroy_kustomization.contains("02-rbac/amber-provisioner-role.yaml")); - assert!(!destroy_kustomization.contains("02-rbac/amber-provisioner-rolebinding.yaml")); - assert!(!destroy_kustomization.contains("02-rbac/amber-provisioner-sa.yaml")); } #[test] -fn kubernetes_peer_addrs_are_derived_from_mesh_secret_names() { - let temp = tempdir().expect("tempdir should be created"); - let configmaps_dir = temp.path().join("01-configmaps"); - fs::create_dir_all(&configmaps_dir).expect("configmaps dir should exist"); - - let mesh_plan = MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/job-compose/kind_helper".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 23000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: Vec::new(), - inbound: Vec::new(), - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::KubernetesSecret { - name: "c7-kind-helper-mesh".to_string(), - namespace: None, - }, - }], - }; - let plan_json = serde_json::to_string_pretty(&mesh_plan).expect("mesh plan should serialize"); - let indented_json = plan_json - .lines() - .map(|line| format!(" {line}")) - .collect::>() - .join("\n"); - fs::write( - configmaps_dir.join("amber-mesh-provision.yaml"), - format!( - "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: amber-mesh-provision\ndata:\n \ - mesh-plan.json: |-\n{indented_json}\n" - ), - ) - .expect("mesh provision configmap should be written"); - - let peer_addrs = - kubernetes_peer_addrs_for_artifact(temp.path()).expect("peer addrs should resolve"); - assert_eq!( - peer_addrs - .get("/job-compose/kind_helper") - .map(String::as_str), - Some("c7-kind-helper:23000"), - ); -} - -#[test] -fn kubernetes_dynamic_route_overlay_uses_runtime_plan_with_router_target() { - let temp = tempdir().expect("tempdir should be created"); - let configmaps_dir = temp.path().join("01-configmaps"); - fs::create_dir_all(&configmaps_dir).expect("configmaps dir should exist"); - - let component_target = amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Component, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/job-compose/kind_helper".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 23000)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: Vec::new(), - inbound: Vec::new(), - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::KubernetesSecret { - name: "c7-kind-helper-mesh".to_string(), - namespace: None, - }, - }; - let embedded_plan = MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![component_target.clone()], - }; - let runtime_plan = MeshProvisionPlan { - version: "2".to_string(), - identity_seed: None, - existing_peer_identities: Vec::new(), - targets: vec![ - component_target, - amber_mesh::MeshProvisionTarget { - kind: MeshProvisionTargetKind::Router, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/site/kind_local/router".to_string(), - mesh_scope: Some("scope".to_string()), - }, - mesh_listen: SocketAddr::from(([127, 0, 0, 1], 23001)), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: Vec::new(), - inbound: Vec::new(), - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, - }, - output: MeshProvisionOutput::KubernetesSecret { - name: "amber-router-mesh".to_string(), - namespace: None, - }, - }, - ], - }; - let embedded_json = - serde_json::to_string_pretty(&embedded_plan).expect("embedded plan should serialize"); - let indented_json = embedded_json - .lines() - .map(|line| format!(" {line}")) - .collect::>() - .join("\n"); - fs::write( - configmaps_dir.join("amber-mesh-provision.yaml"), - format!( - "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: amber-mesh-provision\ndata:\n \ - mesh-plan.json: |-\n{indented_json}\n" - ), - ) - .expect("mesh provision configmap should be written"); - write_json(&temp.path().join("mesh-provision-plan.json"), &runtime_plan) - .expect("runtime plan should be written"); - - build_kubernetes_route_overlay_base( - temp.path(), - &["/job-compose/kind_helper".to_string()], - &BTreeMap::new(), - &BTreeMap::new(), - ) - .expect("runtime plan should supply the router target"); -} - -#[test] -fn kubernetes_expected_workloads_reads_current_artifact_names() { - let temp = tempdir().expect("tempdir should be created"); - let artifact_root = temp.path(); - fs::create_dir_all(artifact_root.join("03-deployments")) - .expect("deployment dir should be created"); - fs::create_dir_all(artifact_root.join("02-rbac")).expect("rbac dir should be created"); - fs::write( - artifact_root.join("03-deployments").join("app.yaml"), - r#" -apiVersion: apps/v1 -kind: Deployment -metadata: - name: child-helper ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: amber-router -"#, - ) - .expect("deployment yaml should be written"); - fs::write( - artifact_root.join("02-rbac").join("job.yaml"), - r#" -apiVersion: batch/v1 -kind: Job -metadata: - name: amber-provisioner-1234 -"#, - ) - .expect("job yaml should be written"); - - let workloads = supervisor::kubernetes_expected_workloads(artifact_root) - .expect("artifact workloads should parse"); - assert_eq!(workloads.jobs, vec!["amber-provisioner-1234"]); - assert_eq!(workloads.deployments, vec!["amber-router", "child-helper"]); -} - -#[test] -fn cleanup_dynamic_site_children_removes_child_roots_and_clears_state() { - let temp = tempdir().expect("tempdir should be created"); - let site_state_root = temp.path().join("state").join("direct_local"); - let child_root = site_controller_runtime_child_root_for_site(&site_state_root, 7); - fs::create_dir_all(child_root.join("artifact")).expect("child artifact dir should exist"); - fs::write(child_root.join("artifact").join("marker.txt"), "marker") - .expect("child marker should be written"); - write_json( - &site_controller_runtime_state_path(&site_state_root), - &SiteControllerRuntimeState { - schema: "amber.run.site_controller_runtime_state".to_string(), - version: 1, - run_id: "run-123".to_string(), - site_id: "direct_local".to_string(), - kind: SiteKind::Direct, - children: BTreeMap::from([( - 7, - SiteControllerRuntimeChildRecord { - child_id: 7, - artifact_root: child_root.join("artifact").display().to_string(), - assigned_components: Vec::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), - process_pid: None, - published: true, - }, - )]), - }, - ) - .expect("site controller runtime state should be written"); - - cleanup_dynamic_site_children(&site_state_root, SiteKind::Direct) - .expect("dynamic site children should be cleaned"); - - let state: SiteControllerRuntimeState = read_json( - &site_controller_runtime_state_path(&site_state_root), - "site controller runtime state", - ) - .expect("site controller runtime state should be readable"); - assert!(state.children.is_empty()); - assert!(!child_root.exists()); -} - -#[test] -fn parse_process_table_reads_ps_output() { - assert_eq!( - parse_process_table(" 42 1\n 84 42\n").expect("process table should parse"), - HashMap::from([(42, 1), (84, 42)]) - ); -} - -#[cfg(unix)] -#[test] -fn parse_process_status_code_reads_ps_state() { - assert_eq!(parse_process_status_code("S+\n"), Some('S')); - assert_eq!(parse_process_status_code("z\n"), Some('Z')); - assert_eq!(parse_process_status_code("\n"), None); -} - -#[cfg(unix)] -#[test] -fn collect_process_tree_postorder_visits_descendants_before_parent() { - let children_by_parent = HashMap::from([(1, vec![2, 3]), (2, vec![4]), (3, vec![5, 6])]); - let mut ordered = Vec::new(); - collect_process_tree_postorder(1, &children_by_parent, &mut ordered); - assert_eq!(ordered, vec![4, 2, 5, 6, 3, 1]); -} - -#[test] -fn container_host_from_resolved_ip_matches_provider_and_consumer_kind() { - assert_eq!( - container_host_from_resolved_ip(SiteKind::Compose, SiteKind::Direct, Some("172.17.0.1"),), - "127.0.0.1" - ); - assert_eq!( - container_host_from_resolved_ip(SiteKind::Compose, SiteKind::Vm, Some("172.17.0.1"),), - "127.0.0.1" - ); - assert_eq!( - container_host_from_resolved_ip(SiteKind::Vm, SiteKind::Compose, Some("172.17.0.1"),), - CONTAINER_HOST_ALIAS - ); - assert_eq!( - container_host_from_resolved_ip(SiteKind::Direct, SiteKind::Compose, Some("172.17.0.1"),), - CONTAINER_HOST_ALIAS - ); - assert_eq!( - container_host_from_resolved_ip( - SiteKind::Kubernetes, - SiteKind::Compose, - Some("172.17.0.1"), - ), - "172.17.0.1" - ); - assert_eq!( - container_host_from_resolved_ip(SiteKind::Vm, SiteKind::Kubernetes, Some("172.17.0.1"),), - "172.17.0.1" - ); - assert_eq!( - container_host_from_resolved_ip(SiteKind::Vm, SiteKind::Kubernetes, None), - CONTAINER_HOST_ALIAS - ); -} - -#[test] -fn containerized_consumers_bridge_runtime_links() { - assert!(link_needs_bridge_proxy(SiteKind::Direct, SiteKind::Compose)); - assert!(link_needs_bridge_proxy(SiteKind::Vm, SiteKind::Compose)); - assert!(!link_needs_bridge_proxy( - SiteKind::Kubernetes, - SiteKind::Compose - )); - assert!(link_needs_bridge_proxy( - SiteKind::Direct, - SiteKind::Kubernetes - )); - assert!(link_needs_bridge_proxy(SiteKind::Vm, SiteKind::Kubernetes)); - assert!(!link_needs_bridge_proxy( - SiteKind::Kubernetes, - SiteKind::Kubernetes - )); - assert!(link_needs_bridge_proxy( - SiteKind::Compose, - SiteKind::Compose - )); - assert!(!link_needs_bridge_proxy(SiteKind::Vm, SiteKind::Direct)); -} - -#[test] -fn bridge_proxy_bind_and_probe_addresses_match_consumer_kind() { - let compose_listen = bridge_proxy_bind_addr(SiteKind::Compose, 41000); - assert_eq!(compose_listen, SocketAddr::from(([0, 0, 0, 0], 41000))); - assert_eq!( - bridge_proxy_probe_addr(compose_listen), - SocketAddr::from(([127, 0, 0, 1], 41000)) - ); - - let kind_listen = bridge_proxy_bind_addr(SiteKind::Kubernetes, 42000); - assert_eq!(kind_listen, SocketAddr::from(([0, 0, 0, 0], 42000))); - assert_eq!( - bridge_proxy_probe_addr(kind_listen), - SocketAddr::from(([127, 0, 0, 1], 42000)) - ); - - let direct_listen = bridge_proxy_bind_addr(SiteKind::Direct, 43000); - assert_eq!(direct_listen, SocketAddr::from(([127, 0, 0, 1], 43000))); - assert_eq!(bridge_proxy_probe_addr(direct_listen), direct_listen); -} - -#[test] -fn host_service_addressing_matches_consumer_kind() { - assert_eq!( - host_service_bind_addr_for_consumer(SiteKind::Compose, 41000), - SocketAddr::from(([0, 0, 0, 0], 41000)) - ); - assert_eq!( - host_service_bind_addr_for_consumer(SiteKind::Kubernetes, 42000), - SocketAddr::from(([0, 0, 0, 0], 42000)) - ); - assert_eq!( - host_service_bind_addr_for_consumer(SiteKind::Direct, 43000), - SocketAddr::from(([127, 0, 0, 1], 43000)) - ); - assert_eq!( - host_service_bind_addr_for_consumer(SiteKind::Vm, 44000), - SocketAddr::from(([127, 0, 0, 1], 44000)) - ); - assert_eq!( - host_service_host_for_consumer(SiteKind::Compose), - "host.docker.internal" - ); - assert_eq!( - host_service_host_for_consumer(SiteKind::Direct), - "127.0.0.1" - ); - assert_eq!(host_service_host_for_consumer(SiteKind::Vm), "127.0.0.1"); - assert_eq!( - host_service_host_for_consumer(SiteKind::Kubernetes), - container_host_for_consumer(SiteKind::Direct, SiteKind::Kubernetes) - ); -} +fn site_controller_image_override_uses_dev_tag() { + let overrides = BTreeMap::from([( + "site_controller".to_string(), + "dev-site-controller".to_string(), + )]); -#[test] -fn bridge_proxy_external_url_uses_consumer_aware_host() { - assert_eq!( - bridge_proxy_external_url(44000, NetworkProtocol::Http, SiteKind::Compose) - .expect("http bridge proxy url should be valid"), - "http://host.docker.internal:44000" - ); assert_eq!( - bridge_proxy_external_url(45000, NetworkProtocol::Http, SiteKind::Kubernetes) - .expect("http bridge proxy url should be valid"), + launch_bundle::site_controller_image_reference_from_overrides(&overrides), format!( - "http://{}:45000", - host_service_host_for_consumer(SiteKind::Kubernetes) + "{}/{}:{}", + amber_images::AMBER_SITE_CONTROLLER.registry, + amber_images::AMBER_SITE_CONTROLLER.name, + "dev-site-controller", ) ); } - -#[test] -fn router_mesh_addr_for_container_consumers_uses_projected_host() { - assert_eq!( - router_mesh_addr_for_consumer(SiteKind::Direct, SiteKind::Compose, "127.0.0.1:24000") - .expect("compose consumer router addr should be valid"), - "host.docker.internal:24000" - ); - assert_eq!( - router_mesh_addr_for_consumer(SiteKind::Direct, SiteKind::Kubernetes, "127.0.0.1:25000") - .expect("kubernetes consumer router addr should be valid"), - format!( - "{}:25000", - container_host_for_consumer(SiteKind::Direct, SiteKind::Kubernetes) - ) - ); -} - -#[test] -fn router_mesh_addr_for_local_consumers_preserves_existing_port() { - let rewritten = - router_mesh_addr_for_consumer(SiteKind::Direct, SiteKind::Direct, "127.0.0.1:26000") - .expect("local consumer router addr should be valid"); - #[cfg(target_os = "linux")] - assert_eq!(rewritten, "10.0.2.2:26000"); - #[cfg(not(target_os = "linux"))] - assert_eq!(rewritten, "127.0.0.1:26000"); -} - -#[test] -fn outside_proxy_mesh_listener_stays_loopback_for_local_consumers() { - let context = RunOutsideProxyContext { - mesh_scope: "scope".to_string(), - sites: BTreeMap::from([( - "direct".to_string(), - test_launched_site_with_kind(SiteKind::Direct), - )]), - exports: BTreeMap::new(), - slots: BTreeMap::from([( - "api".to_string(), - RunOutsideSlot { - required: true, - kind: CapabilityKind::Http, - url_env: "AMBER_EXTERNAL_SLOT_API_URL".to_string(), - consumer_sites: vec!["direct".to_string()], - }, - )]), - }; - - assert_eq!( - outside_proxy_mesh_listen_addr( - &context, - &[("api".to_string(), "http://127.0.0.1:9000".to_string())], - 48000, - ) - .expect("outside proxy bind addr"), - SocketAddr::from(([127, 0, 0, 1], 48000)) - ); -} - -#[test] -fn outside_proxy_mesh_listener_expands_for_container_consumers() { - let context = RunOutsideProxyContext { - mesh_scope: "scope".to_string(), - sites: BTreeMap::from([ - ( - "direct".to_string(), - test_launched_site_with_kind(SiteKind::Direct), - ), - ( - "compose".to_string(), - test_launched_site_with_kind(SiteKind::Compose), - ), - ]), - exports: BTreeMap::new(), - slots: BTreeMap::from([( - "api".to_string(), - RunOutsideSlot { - required: true, - kind: CapabilityKind::Http, - url_env: "AMBER_EXTERNAL_SLOT_API_URL".to_string(), - consumer_sites: vec!["direct".to_string(), "compose".to_string()], - }, - )]), - }; - - assert_eq!( - outside_proxy_mesh_listen_addr( - &context, - &[("api".to_string(), "http://127.0.0.1:9000".to_string())], - 49000, - ) - .expect("outside proxy bind addr"), - SocketAddr::from(([0, 0, 0, 0], 49000)) - ); -} - -#[test] -fn bridge_proxy_export_binding_uses_selected_listen_addr() { - assert_eq!( - bridge_proxy_export_binding("api", SocketAddr::from(([127, 0, 0, 1], 46000))), - "api=127.0.0.1:46000" - ); - assert_eq!( - bridge_proxy_export_binding("api", SocketAddr::from(([0, 0, 0, 0], 47000))), - "api=0.0.0.0:47000" - ); -} - -fn test_launched_site_with_kind(kind: SiteKind) -> LaunchedSite { - LaunchedSite { - receipt: SiteReceipt { - kind, - artifact_dir: "/tmp/artifact".to_string(), - supervisor_pid: 1, - process_pid: None, - compose_project: None, - context: None, - kubernetes_namespace: None, - port_forward_pid: None, - router_mesh_addr: None, - router_control: None, - router_identity_id: None, - router_public_key_b64: None, - site_controller_pid: None, - site_controller_url: None, - }, - router_identity: MeshIdentityPublic { - id: format!("/site/{kind:?}"), - public_key: [0; 32], - mesh_scope: None, - }, - router_addr: SocketAddr::from(([127, 0, 0, 1], 24000)), - router_control: ControlEndpoint::Tcp("127.0.0.1:24100".to_string()), - } -} - -#[test] -fn kubernetes_sites_get_startup_budget_after_workloads_are_ready() { - assert_eq!( - site_ready_timeout_for_kind(SiteKind::Kubernetes), - KUBERNETES_WORKLOAD_READY_TIMEOUT + KUBERNETES_SITE_READY_BUFFER - ); -} - -#[test] -fn kubernetes_namespace_name_is_run_scoped() { - assert_eq!( - kubernetes_namespace_name("run-1234abcd", "kind_c"), - "amber-run-1234abcd-kind-c" - ); - assert_ne!( - kubernetes_namespace_name("run-1234abcd", "kind_c"), - kubernetes_namespace_name("run-5678efgh", "kind_c") - ); -} - -#[test] -fn prepare_kubernetes_artifact_namespace_rewrites_kustomization_namespace() { - let temp = tempdir().expect("tempdir should be created"); - let kustomization = temp.path().join("kustomization.yaml"); - fs::write( - &kustomization, - "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nnamespace: \ - scenario-old\n", - ) - .expect("kustomization should be written"); - - let namespace = prepare_kubernetes_artifact_namespace("run-1234abcd", "kind_c", temp.path()) - .expect("artifact namespace should be prepared"); - - assert_eq!(namespace, "amber-run-1234abcd-kind-c"); - assert_eq!( - fs::read_to_string(&kustomization).expect("kustomization should be readable"), - "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nnamespace: \ - amber-run-1234abcd-kind-c\n" - ); -} - -#[test] -fn prepare_kubernetes_site_artifact_for_apply_rewrites_runtime_artifact_state() { - let temp = tempdir().expect("tempdir should be created"); - let kustomization = temp.path().join("kustomization.yaml"); - fs::write( - &kustomization, - "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nnamespace: \ - scenario-old\n", - ) - .expect("kustomization should be written"); - let env_file = temp.path().join("root-config.env"); - fs::write(&env_file, "AMBER_TEST_VALUE=stale\n").expect("env file should be written"); - - let mut launch_env = BTreeMap::new(); - launch_env.insert("AMBER_TEST_VALUE".to_string(), "fresh".to_string()); - let plan = SiteControllerRuntimePlan { - schema: SITE_CONTROLLER_RUNTIME_PLAN_SCHEMA.to_string(), - version: SITE_PLAN_VERSION, - run_id: "run-1234abcd".to_string(), - mesh_scope: "mesh-scope-test".to_string(), - run_root: temp.path().display().to_string(), - site_id: "kind_c".to_string(), - kind: SiteKind::Kubernetes, - router_identity_id: "/site/kind_c/router".to_string(), - artifact_dir: temp.path().display().to_string(), - site_state_root: temp.path().join("state").display().to_string(), - listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: None, - kubernetes_namespace: None, - context: Some("kind-test".to_string()), - observability_endpoint: None, - launch_env, - }; - - let supervisor_plan = prepare_kubernetes_artifact_for_apply(&plan, temp.path()) - .expect("artifact should be prepared"); - - assert_eq!( - supervisor_plan.kubernetes_namespace.as_deref(), - Some("amber-run-1234abcd-kind-c") - ); - assert_eq!( - fs::read_to_string(&kustomization).expect("kustomization should be readable"), - "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nnamespace: \ - amber-run-1234abcd-kind-c\n" - ); - assert_eq!( - fs::read_to_string(&env_file).expect("env file should be readable"), - "AMBER_TEST_VALUE=fresh\n" - ); -} - -#[test] -fn external_slot_name_from_env_var_restores_slot_name() { - assert_eq!( - external_slot_name_from_env_var("AMBER_EXTERNAL_SLOT_API_URL"), - "api" - ); -} - -#[test] -fn external_slot_env_for_site_skips_missing_weak_provider() { - let env = external_slot_env_for_site( - "consumer_site", - SiteKind::Direct, - &[RunLink { - provider_site: "provider_site".to_string(), - consumer_site: "consumer_site".to_string(), - provider_component: "/provider".to_string(), - provide: "api".to_string(), - consumer_component: "/consumer".to_string(), - slot: "upstream".to_string(), - weak: true, - protocol: NetworkProtocol::Http, - export_name: "amber_export_provider_api_http".to_string(), - external_slot_name: "amber_link_consumer_provider_api_http".to_string(), - }], - &BTreeMap::new(), - ) - .expect("weak links should not require a launched provider"); - assert!(env.is_empty()); -} - -#[test] -fn maybe_resolve_proxy_run_target_resolves_run_id_and_prefers_live_state() { - let temp = tempdir().expect("tempdir should exist"); - let storage_root = temp.path(); - let run_id = "run-123"; - let run_root = storage_root.join("runs").join(run_id); - let artifact_dir = run_root.join("sites").join("direct_local").join("artifact"); - fs::create_dir_all(&artifact_dir).expect("artifact dir should exist"); - let state_root = run_root.join("state"); - fs::create_dir_all(state_root.join("direct_local")).expect("state dir should exist"); - - let receipt = RunReceipt { - schema: RECEIPT_SCHEMA.to_string(), - version: RECEIPT_VERSION, - run_id: run_id.to_string(), - mesh_scope: "mesh.scope.test".to_string(), - plan_path: run_plan_path(&run_root).display().to_string(), - source_plan_path: None, - run_root: run_root.display().to_string(), - observability: None, - bridge_proxies: Vec::new(), - sites: BTreeMap::from([( - "direct_local".to_string(), - test_site_receipt( - SiteKind::Direct, - &artifact_dir, - Some("unix:///receipt.sock"), - Some("127.0.0.1:18080"), - ), - )]), - }; - write_json(&receipt_path(&run_root), &receipt).expect("receipt should serialize"); - write_json( - &site_state_path(&state_root, "direct_local"), - &test_site_state( - run_id, - "direct_local", - SiteKind::Direct, - &artifact_dir, - Some("unix:///live.sock"), - Some("127.0.0.1:18081"), - ), - ) - .expect("state should serialize"); - - let resolved = maybe_resolve_proxy_run_target(run_id, Some("direct_local"), Some(storage_root)) - .expect("run target resolution should succeed") - .expect("run target should resolve"); - - assert_eq!( - resolved.artifact_dir, - artifact_dir - .canonicalize() - .expect("artifact dir should canonicalize") - ); - assert_eq!( - resolved.router_control_addr.as_deref(), - Some("unix:///live.sock") - ); - assert_eq!( - resolved.router_addr, - Some( - "127.0.0.1:18081" - .parse::() - .expect("socket addr should parse") - ) - ); -} - -#[test] -fn maybe_resolve_proxy_run_target_requires_site_for_multi_site_run() { - let temp = tempdir().expect("tempdir should exist"); - let storage_root = temp.path(); - let run_id = "run-456"; - let run_root = storage_root.join("runs").join(run_id); - let direct_artifact = run_root.join("sites").join("direct_local").join("artifact"); - let compose_artifact = run_root - .join("sites") - .join("compose_local") - .join("artifact"); - fs::create_dir_all(&direct_artifact).expect("direct artifact dir should exist"); - fs::create_dir_all(&compose_artifact).expect("compose artifact dir should exist"); - - let receipt = RunReceipt { - schema: RECEIPT_SCHEMA.to_string(), - version: RECEIPT_VERSION, - run_id: run_id.to_string(), - mesh_scope: "mesh.scope.test".to_string(), - plan_path: run_plan_path(&run_root).display().to_string(), - source_plan_path: None, - run_root: run_root.display().to_string(), - observability: None, - bridge_proxies: Vec::new(), - sites: BTreeMap::from([ - ( - "compose_local".to_string(), - test_site_receipt( - SiteKind::Compose, - &compose_artifact, - Some("unix:///compose.sock"), - Some("127.0.0.1:19090"), - ), - ), - ( - "direct_local".to_string(), - test_site_receipt( - SiteKind::Direct, - &direct_artifact, - Some("unix:///direct.sock"), - Some("127.0.0.1:19091"), - ), - ), - ]), - }; - write_json(&receipt_path(&run_root), &receipt).expect("receipt should serialize"); - - let err = maybe_resolve_proxy_run_target(run_id, None, Some(storage_root)) - .expect_err("multi-site run ids should require --site"); - let message = err.to_string(); - assert!( - message.contains("contains multiple sites"), - "expected multi-site guidance, got: {message}" - ); - assert!( - message.contains("--site "), - "expected --site guidance, got: {message}" - ); -} - -#[cfg(unix)] -#[tokio::test] -async fn stop_run_forces_supervisor_shutdown_and_cleans_up() { - let temp = tempdir().expect("tempdir should exist"); - let storage_root = temp.path(); - let run_id = "run-stuck"; - let run_root = storage_root.join("runs").join(run_id); - let artifact_dir = run_root.join("sites").join("direct_local").join("artifact"); - fs::create_dir_all(&artifact_dir).expect("artifact dir should exist"); - - let mut stubborn_supervisor = Command::new("sh") - .arg("-c") - .arg("trap '' TERM; while :; do sleep 1 & wait $!; done") - .spawn() - .expect("stubborn supervisor should spawn"); - - let receipt = RunReceipt { - schema: RECEIPT_SCHEMA.to_string(), - version: RECEIPT_VERSION, - run_id: run_id.to_string(), - mesh_scope: "mesh.scope.test".to_string(), - plan_path: run_plan_path(&run_root).display().to_string(), - source_plan_path: None, - run_root: run_root.display().to_string(), - observability: None, - bridge_proxies: Vec::new(), - sites: BTreeMap::from([( - "direct_local".to_string(), - SiteReceipt { - supervisor_pid: stubborn_supervisor.id(), - ..test_site_receipt( - SiteKind::Direct, - &artifact_dir, - Some("unix:///receipt.sock"), - Some("127.0.0.1:18080"), - ) - }, - )]), - }; - write_json(&receipt_path(&run_root), &receipt).expect("receipt should serialize"); - - let state_root = run_root.join("state"); - let mut state = test_site_state( - run_id, - "direct_local", - SiteKind::Direct, - &artifact_dir, - Some("unix:///live.sock"), - Some("127.0.0.1:18081"), - ); - state.status = SiteLifecycleStatus::Stopped; - state.supervisor_pid = stubborn_supervisor.id(); - write_json(&site_state_path(&state_root, "direct_local"), &state) - .expect("state should serialize"); - - let result = stop_run(run_id, Some(storage_root)).await; - - let _ = stubborn_supervisor.kill(); - let _ = stubborn_supervisor.wait(); - - result.expect("stop_run should force the supervisor down and succeed"); - assert!( - !receipt_path(&run_root).is_file(), - "receipt should be removed after forced shutdown cleanup" - ); - assert!( - stop_marker_path(&run_root).is_file(), - "stop marker should be written for supervisors" - ); - - let updated_state: SiteManagerState = read_json( - &site_state_path(&state_root, "direct_local"), - "site manager state", - ) - .expect("updated state should deserialize"); - assert_eq!(updated_state.status, SiteLifecycleStatus::Stopped); - assert!( - updated_state.last_error.as_deref().is_some_and(|value| { - value.contains("forcing shutdown") || value.contains("exited before confirming stop") - }), - "expected escalated shutdown cleanup to be recorded, got: {:?}", - updated_state.last_error - ); -} - -#[cfg(unix)] -#[tokio::test] -async fn stop_bridge_proxies_terminates_dynamic_children() { - let temp = tempdir().expect("tempdir should exist"); - let child = Command::new("sh") - .current_dir(temp.path()) - .arg("-c") - .arg("sleep 30") - .spawn() - .expect("bridge proxy child should spawn"); - let pid = child.id(); - - let mut bridge_proxies = BTreeMap::from([( - BridgeProxyKey { - provider_output_dir: temp.path().display().to_string(), - export_name: "amber_export_test".to_string(), - consumer_kind: SiteKind::Compose, - }, - BridgeProxyHandle { - child, - export_name: "amber_export_test".to_string(), - listen: SocketAddr::from(([127, 0, 0, 1], 46000)), - }, - )]); - - stop_bridge_proxies(&mut bridge_proxies) - .await - .expect("bridge proxies should stop cleanly"); - - assert!( - bridge_proxies.is_empty(), - "bridge proxy cleanup should remove all tracked proxies" - ); - assert!( - !pid_is_alive(pid), - "bridge proxy cleanup should terminate the owned child process" - ); -} - -#[tokio::test] -async fn wait_for_kubernetes_site_router_ready_waits_for_live_discovery() { - let temp = tempdir().expect("tempdir should exist"); - let site_state_root = temp.path().join("state").join("kind_local"); - let artifact_dir = temp.path().join("artifact"); - fs::create_dir_all(&site_state_root).expect("site state root should exist"); - fs::create_dir_all(&artifact_dir).expect("artifact dir should exist"); - - let stale_control = - TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).expect("stale control"); - let stale_control_addr = stale_control.local_addr().expect("stale control addr"); - drop(stale_control); - let stale_mesh = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).expect("stale mesh"); - let stale_mesh_addr = stale_mesh.local_addr().expect("stale mesh addr"); - drop(stale_mesh); - - write_json( - &site_state_root.join("manager-state.json"), - &test_site_state( - "run-test", - "kind_local", - SiteKind::Kubernetes, - &artifact_dir, - Some(&stale_control_addr.to_string()), - Some(&stale_mesh_addr.to_string()), - ), - ) - .expect("initial site manager state should serialize"); - - let live_control = tokio::net::TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) - .await - .expect("live control listener"); - let live_control_addr = live_control.local_addr().expect("live control addr"); - let live_mesh = tokio::net::TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) - .await - .expect("live mesh listener"); - let live_mesh_addr = live_mesh.local_addr().expect("live mesh addr"); - - let identity_body = serde_json::to_string(&MeshIdentityPublic { - id: "/site/kind_local/router".to_string(), - public_key: [7; 32], - mesh_scope: Some("mesh.scope.test".to_string()), - }) - .expect("identity body should serialize"); - let identity_response = format!( - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - identity_body.len(), - identity_body - ); - - let control_task = tokio::spawn(async move { - let (mut stream, _) = live_control.accept().await.expect("control accept"); - let mut request = [0u8; 1024]; - let _ = stream.read(&mut request).await.expect("control read"); - stream - .write_all(identity_response.as_bytes()) - .await - .expect("control response should write"); - }); - let mesh_task = tokio::spawn(async move { - let _ = live_mesh.accept().await.expect("mesh accept"); - }); - let state_update_task = tokio::spawn({ - let artifact_dir = artifact_dir.clone(); - let site_state_root = site_state_root.clone(); - async move { - sleep(Duration::from_millis(300)).await; - write_json( - &site_state_root.join("manager-state.json"), - &test_site_state( - "run-test", - "kind_local", - SiteKind::Kubernetes, - &artifact_dir, - Some(&live_control_addr.to_string()), - Some(&live_mesh_addr.to_string()), - ), - ) - .expect("updated site manager state should serialize"); - } - }); - - let started = Instant::now(); - wait_for_kubernetes_site_router_ready( - &SiteControllerRuntimePlan { - schema: SITE_CONTROLLER_RUNTIME_PLAN_SCHEMA.to_string(), - version: SITE_CONTROLLER_RUNTIME_PLAN_VERSION, - run_id: "run-test".to_string(), - mesh_scope: "mesh.scope.test".to_string(), - run_root: temp.path().display().to_string(), - site_id: "kind_local".to_string(), - kind: SiteKind::Kubernetes, - router_identity_id: "/site/kind_local/router".to_string(), - artifact_dir: artifact_dir.display().to_string(), - site_state_root: site_state_root.display().to_string(), - listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), - storage_root: None, - runtime_root: None, - router_mesh_port: None, - compose_project: None, - kubernetes_namespace: Some("amber-test".to_string()), - context: Some("kind-test".to_string()), - observability_endpoint: None, - launch_env: BTreeMap::new(), - }, - Duration::from_secs(5), - ) - .await - .expect("router readiness wait should succeed once discovery updates"); - assert!( - started.elapsed() >= Duration::from_millis(300), - "router readiness wait should not return while manager state still points at the stale \ - endpoint" - ); - - state_update_task - .await - .expect("state update task should join"); - control_task.await.expect("control task should join"); - mesh_task.await.expect("mesh task should join"); -} - -#[test] -fn handle_otlp_connection_persists_structured_trace_stream() { - let temp = tempdir().expect("tempdir should exist"); - let requests_log = temp.path().join("observability").join("requests.log"); - let events_ndjson = temp.path().join("observability").join("events.ndjson"); - let body = ExportLogsServiceRequest { - resource_logs: vec![ResourceLogs { - resource: Some(Resource { - attributes: vec![ - KeyValue { - key: "service.name".to_string(), - value: Some(otlp_string_value("amber.run-test.bindings")), - }, - KeyValue { - key: "amber_entity_kind".to_string(), - value: Some(otlp_string_value("binding")), - }, - KeyValue { - key: "amber_edge_ref".to_string(), - value: Some(otlp_string_value("/public.a2a -> /server.a2a")), - }, - KeyValue { - key: "amber_http_subject".to_string(), - value: Some(otlp_string_value("agent card")), - }, - ], - ..Default::default() - }), - scope_logs: vec![ScopeLogs { - scope: Some(InstrumentationScope { - name: "amber.binding".to_string(), - ..Default::default() - }), - log_records: vec![LogRecord { - severity_text: "INFO".to_string(), - body: Some(otlp_string_value( - "request received from public by /server [headers]", - )), - trace_id: vec![0x1a; 16], - ..Default::default() - }], - ..Default::default() - }], - ..Default::default() - }], - } - .encode_to_vec(); - - let listener = TcpListener::bind(("127.0.0.1", 0)).expect("listener should bind"); - let addr = listener - .local_addr() - .expect("listener should expose address"); - let response_task = std::thread::spawn(move || { - let mut client = TcpStream::connect(addr).expect("client should connect"); - write!( - client, - "POST /v1/logs HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Length: {}\r\nConnection: \ - close\r\n\r\n", - body.len() - ) - .expect("request head should write"); - client.write_all(&body).expect("request body should write"); - let mut response = String::new(); - client - .read_to_string(&mut response) - .expect("response should read"); - response - }); - - let (mut stream, _) = listener.accept().expect("listener should accept"); - handle_otlp_connection(&mut stream, &requests_log, &events_ndjson) - .expect("sink should handle otlp request"); - drop(stream); - - let response = response_task.join().expect("client thread should join"); - assert!( - response.starts_with("HTTP/1.1 204 No Content"), - "unexpected response: {response}" - ); - assert!( - fs::read_to_string(&requests_log) - .expect("requests log should exist") - .contains("/v1/logs"), - "requests log should record the OTLP path" - ); - let structured = fs::read_to_string(&events_ndjson).expect("structured events should exist"); - assert!( - structured.contains("\"message\":\"request received from public by /server [headers]\""), - "structured log should contain the event body, got:\n{structured}" - ); - assert!( - structured.contains("\"amber_http_subject\":\"agent card\""), - "structured log should preserve protocol attributes, got:\n{structured}" - ); -} - -#[test] -fn handle_otlp_connection_ignores_internal_and_json_log_payloads() { - let temp = tempdir().expect("tempdir should exist"); - let requests_log = temp.path().join("observability").join("requests.log"); - let events_ndjson = temp.path().join("observability").join("events.ndjson"); - - let internal_body = ExportLogsServiceRequest { - resource_logs: vec![ResourceLogs { - resource: Some(Resource { - attributes: vec![ - KeyValue { - key: "amber_entity_kind".to_string(), - value: Some(otlp_string_value("binding")), - }, - KeyValue { - key: "amber_edge_ref".to_string(), - value: Some(otlp_string_value( - "/site/compose_local/router.a2a -> /runtime.a2a", - )), - }, - ], - ..Default::default() - }), - scope_logs: vec![ScopeLogs { - scope: Some(InstrumentationScope { - name: "amber.binding".to_string(), - ..Default::default() - }), - log_records: vec![LogRecord { - severity_text: "INFO".to_string(), - body: Some(otlp_string_value( - "request received by /runtime from /site/compose_local/router via \ - /site/compose_local/router.a2a -> /runtime.a2a [headers]", - )), - trace_id: vec![0x2b; 16], - ..Default::default() - }], - ..Default::default() - }], - ..Default::default() - }], - } - .encode_to_vec(); - - let listener = TcpListener::bind(("127.0.0.1", 0)).expect("listener should bind"); - let addr = listener - .local_addr() - .expect("listener should expose address"); - let internal_request = std::thread::spawn(move || { - let mut client = TcpStream::connect(addr).expect("client should connect"); - write!( - client, - "POST /v1/logs HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Length: {}\r\nConnection: \ - close\r\n\r\n", - internal_body.len() - ) - .expect("request head should write"); - client - .write_all(&internal_body) - .expect("request body should write"); - let mut response = String::new(); - client - .read_to_string(&mut response) - .expect("response should read"); - response - }); - let (mut stream, _) = listener.accept().expect("listener should accept"); - handle_otlp_connection(&mut stream, &requests_log, &events_ndjson) - .expect("sink should handle internal otlp request"); - drop(stream); - let response = internal_request.join().expect("client thread should join"); - assert!( - response.starts_with("HTTP/1.1 204 No Content"), - "unexpected response: {response}" - ); - assert!( - !events_ndjson.exists(), - "internal route chatter should not produce a structured trace stream" - ); - - let json_body = serde_json::to_vec(&serde_json::json!({ - "event": "amber.run.starting", - "body": "starting mixed-site run run-test", - "attributes": {"amber.run_id": "run-test"} - })) - .expect("json payload should serialize"); - let listener = TcpListener::bind(("127.0.0.1", 0)).expect("listener should bind"); - let addr = listener - .local_addr() - .expect("listener should expose address"); - let json_request = std::thread::spawn(move || { - let mut client = TcpStream::connect(addr).expect("client should connect"); - write!( - client, - "POST /v1/logs HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Type: \ - application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - json_body.len() - ) - .expect("request head should write"); - client - .write_all(&json_body) - .expect("request body should write"); - let mut response = String::new(); - client - .read_to_string(&mut response) - .expect("response should read"); - response - }); - let (mut stream, _) = listener.accept().expect("listener should accept"); - handle_otlp_connection(&mut stream, &requests_log, &events_ndjson) - .expect("sink should handle manager json request"); - drop(stream); - let response = json_request.join().expect("client thread should join"); - assert!( - response.starts_with("HTTP/1.1 204 No Content"), - "unexpected response: {response}" - ); - assert!( - !events_ndjson.exists(), - "manager json events should not be mixed into the structured trace stream" - ); -} diff --git a/cli/src/vm_runtime.rs b/cli/src/vm_runtime.rs index c71201e7..fd7d7bd5 100644 --- a/cli/src/vm_runtime.rs +++ b/cli/src/vm_runtime.rs @@ -54,10 +54,7 @@ mod state; use self::{artifacts::*, preview::*, state::*}; pub(crate) use self::{ preview::build_vm_site_launch_preview, - state::{ - ensure_control_socket_link, vm_current_control_socket_path, - vm_endpoint_forward_ready_timeout, write_vm_runtime_state, - }, + state::{vm_current_control_socket_path, vm_endpoint_forward_ready_timeout}, }; const VM_CHILD_POLL_INTERVAL: Duration = Duration::from_millis(150); diff --git a/cli/src/vm_runtime/state.rs b/cli/src/vm_runtime/state.rs index d063e7e6..47b92072 100644 --- a/cli/src/vm_runtime/state.rs +++ b/cli/src/vm_runtime/state.rs @@ -186,6 +186,14 @@ pub(crate) fn hashed_temp_socket_path(namespace: &str, kind: &str, path: &Path) amber_mesh::stable_temp_socket_path(namespace, kind, path) } +fn vm_component_control_socket_path(work_dir: &Path, component_id: usize) -> PathBuf { + hashed_temp_socket_path( + "amber-vm-control", + &format!("sidecar-{component_id}"), + work_dir, + ) +} + pub(crate) fn assign_vm_runtime_ports( runtime_root: &Path, vm_plan: &VmPlan, @@ -611,6 +619,33 @@ pub(crate) async fn spawn_component_sidecar( work_dir.display() ) })?; + let control_socket_path = vm_component_control_socket_path(&work_dir, component.id); + let control_socket_dir = control_socket_path + .parent() + .ok_or_else(|| miette::miette!("invalid vm sidecar control socket path"))? + .to_path_buf(); + fs::create_dir_all(&control_socket_dir) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to create vm sidecar control directory {}", + control_socket_dir.display() + ) + })?; + if control_socket_path.exists() { + fs::remove_file(&control_socket_path) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to remove stale vm sidecar control socket {}", + control_socket_path.display() + ) + })?; + } + env_map.insert( + "AMBER_ROUTER_CONTROL_SOCKET_PATH".to_string(), + control_socket_path.display().to_string(), + ); spawn_command( format!("{}-sidecar", component.log_name), vec![router_binary.to_string()], @@ -1004,4 +1039,20 @@ mod tests { replacement.component_mesh_port_by_id ); } + + #[test] + fn vm_component_control_socket_path_stays_short_on_long_work_dirs() { + let work_dir = Path::new( + "/Users/example/Developer/amber/target/cli-test-outputs/\ + linux-vm-framework_component-very-long/state/runs/run-123/state/vm_local/runtime/\ + work/sidecars/c2-web", + ); + let socket = vm_component_control_socket_path(work_dir, 2); + let rendered = socket.as_os_str().to_string_lossy(); + + assert!( + rendered.len() < 104, + "vm sidecar control socket path must fit within unix socket limits: {rendered}", + ); + } } diff --git a/cli/tests/linux_vm.rs b/cli/tests/linux_vm.rs index 0992966e..d30df662 100644 --- a/cli/tests/linux_vm.rs +++ b/cli/tests/linux_vm.rs @@ -1016,6 +1016,14 @@ fn linux_vm_runs_framework_component_live_tests() { run_linux_guest_mixed_run_filter("framework_component_").unwrap_or_else(|err| panic!("{err}")); } +#[test] +#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component direct \ + create/destroy live test inside the guest"] +fn linux_vm_runs_framework_component_direct_create_destroy_live() { + run_linux_guest_mixed_run_test("framework_component_direct_create_destroy_live") + .unwrap_or_else(|err| panic!("{err}")); +} + #[test] #[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component bounded \ template replay live test inside the guest"] @@ -1089,6 +1097,14 @@ fn linux_vm_runs_dynamic_capability_live_tests() { run_linux_guest_mixed_run_filter("dynamic_capabilities_").unwrap_or_else(|err| panic!("{err}")); } +#[test] +#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux dynamic-capability dynamic \ + child post-create share live test inside the guest"] +fn linux_vm_runs_dynamic_capabilities_dynamic_child_post_create_share_live() { + run_linux_guest_mixed_run_test("dynamic_capabilities_dynamic_child_post_create_share_live") + .unwrap_or_else(|err| panic!("{err}")); +} + #[test] #[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux dynamic-capability manual \ materialization live test inside the guest"] diff --git a/cli/tests/mixed_run/support.rs b/cli/tests/mixed_run/support.rs index 3c6df9ae..a25336a9 100644 --- a/cli/tests/mixed_run/support.rs +++ b/cli/tests/mixed_run/support.rs @@ -24,7 +24,7 @@ use std::{ time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; -use amber_images::{AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER}; +use amber_images::{AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER, AMBER_SITE_CONTROLLER}; use cloud_image_support::default_host_arch_cloud_image_filename; use outputs_root_support::cli_test_outputs_root; use serde_json::{Value, json}; @@ -691,6 +691,10 @@ pub(crate) fn ensure_amber_internal_images() { AMBER_HELPER.reference, &root.join("docker/amber-helper/Dockerfile"), ); + ensure_docker_image( + AMBER_SITE_CONTROLLER.reference, + &root.join("docker/amber-site-controller/Dockerfile"), + ); }); } @@ -831,6 +835,7 @@ pub(crate) fn ensure_kind_internal_images(kind_cluster: &KindCluster) { load_kind_image(&name, AMBER_ROUTER.reference); load_kind_image(&name, AMBER_PROVISIONER.reference); load_kind_image(&name, AMBER_HELPER.reference); + load_kind_image(&name, AMBER_SITE_CONTROLLER.reference); // The public test workload image is registry-pullable inside the kind cluster, so keep kind // image preloading focused on the Amber images that only exist locally. loaded diff --git a/compiler/src/run_plan.rs b/compiler/src/run_plan.rs index 7d988ead..1bc8a3df 100644 --- a/compiler/src/run_plan.rs +++ b/compiler/src/run_plan.rs @@ -1838,78 +1838,62 @@ fn render_site_artifact_files( mesh_scope: &str, force_router: bool, ) -> Result, RunPlanError> { - let mut files = match site_kind { + let files = match site_kind { SiteKind::Direct => { - crate::targets::direct::emit_direct_artifact(compiled, force_router) - .map_err(|err| RunPlanError::Other(format!("failed to render direct site: {err}")))? - .files + crate::targets::direct::emit_direct_artifact_with_options( + compiled, + crate::targets::direct::DirectArtifactBuildOptions { + force_router, + router_identity_id, + mesh_scope: Some(mesh_scope), + }, + ) + .map_err(|err| RunPlanError::Other(format!("failed to render direct site: {err}")))? + .files } SiteKind::Vm => { - crate::targets::vm::emit_vm_artifact(compiled, force_router) - .map_err(|err| RunPlanError::Other(format!("failed to render vm site: {err}")))? - .files + crate::targets::vm::emit_vm_artifact_with_options( + compiled, + crate::targets::vm::VmArtifactBuildOptions { + force_router, + router_identity_id, + mesh_scope: Some(mesh_scope), + }, + ) + .map_err(|err| RunPlanError::Other(format!("failed to render vm site: {err}")))? + .files } SiteKind::Compose => { - crate::targets::mesh::docker_compose::emit_docker_compose_artifact( + crate::targets::mesh::docker_compose::emit_docker_compose_artifact_with_options( compiled, - force_router, + crate::targets::mesh::docker_compose::DockerComposeArtifactBuildOptions { + force_router, + router_identity_id, + mesh_scope: Some(mesh_scope), + }, ) .map_err(|err| RunPlanError::Other(format!("failed to render compose site: {err}")))? .files } SiteKind::Kubernetes => { - crate::targets::mesh::kubernetes::emit_kubernetes_artifact(compiled, force_router) - .map_err(|err| { - RunPlanError::Other(format!("failed to render kubernetes site: {err}")) - })? - .files + crate::targets::mesh::kubernetes::emit_kubernetes_artifact_with_options( + compiled, + crate::targets::mesh::kubernetes::KubernetesArtifactBuildOptions { + force_router, + router_identity_id, + mesh_scope: Some(mesh_scope), + }, + ) + .map_err(|err| RunPlanError::Other(format!("failed to render kubernetes site: {err}")))? + .files } }; - rewrite_router_identity(site_kind, &mut files, router_identity_id); - rewrite_mesh_scope( - &mut files, - &scenario_mesh_scope(compiled.scenario_ir())?, - mesh_scope, - ); Ok(files .into_iter() .map(|(path, contents)| (path.to_string_lossy().into_owned(), contents)) .collect()) } -fn rewrite_router_identity( - site_kind: SiteKind, - files: &mut BTreeMap, - router_identity_id: &str, -) { - let existing_router_id = match site_kind { - SiteKind::Direct => crate::targets::direct::ROUTER_IDENTITY_ID, - SiteKind::Vm => crate::targets::vm::ROUTER_IDENTITY_ID, - SiteKind::Compose | SiteKind::Kubernetes => { - crate::targets::mesh::mesh_config::DEFAULT_ROUTER_ID - } - }; - if existing_router_id == router_identity_id { - return; - } - for contents in files.values_mut() { - *contents = contents.replace(existing_router_id, router_identity_id); - } -} - -fn rewrite_mesh_scope( - files: &mut BTreeMap, - existing_mesh_scope: &str, - mesh_scope: &str, -) { - if existing_mesh_scope == mesh_scope { - return; - } - for contents in files.values_mut() { - *contents = contents.replace(existing_mesh_scope, mesh_scope); - } -} - #[cfg(test)] mod tests { use std::{ @@ -2617,6 +2601,43 @@ mod tests { } } + #[tokio::test] + async fn compose_site_artifact_keeps_router_control_socket_paths() { + let dir = tmp_dir("run-plan-compose-router-control-"); + let image_child = dir.path().join("image.json5"); + let root = dir.path().join("root.json5"); + + write(&image_child, image_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { + image: "./image.json5" + }, + exports: { + image_api: "#image.api" + } +}"##, + ); + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, None).expect("run plan should build"); + let compose_yaml = plan + .sites + .get("compose_local") + .and_then(|site| site.artifact_files.get("compose.yaml")) + .expect("compose site should have a compose artifact"); + + assert!( + compose_yaml + .contains("AMBER_ROUTER_CONTROL_SOCKET_PATH=/amber/control/router-control.sock") + ); + assert!(compose_yaml.contains("control_socket: /router-control.sock")); + assert!(!compose_yaml.contains("/amber/control/site/compose_local/router-control.sock")); + assert!(!compose_yaml.contains("control_socket: /site/compose_local/router-control.sock")); + } + #[tokio::test] async fn build_unmanaged_export_uses_single_matching_site() { let dir = tmp_dir("run-plan-unmanaged-export-single-"); diff --git a/compiler/src/targets/direct/mod.rs b/compiler/src/targets/direct/mod.rs index fce2f68d..7023f71f 100644 --- a/compiler/src/targets/direct/mod.rs +++ b/compiler/src/targets/direct/mod.rs @@ -63,6 +63,13 @@ pub struct DirectArtifact { pub files: BTreeMap, } +#[derive(Clone, Copy, Debug)] +pub(crate) struct DirectArtifactBuildOptions<'a> { + pub(crate) force_router: bool, + pub(crate) router_identity_id: &'a str, + pub(crate) mesh_scope: Option<&'a str>, +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct DirectPlan { pub version: String, @@ -208,6 +215,20 @@ fn render_direct(compiled: &CompiledScenario) -> Result Result { + emit_direct_artifact_with_options( + compiled, + DirectArtifactBuildOptions { + force_router, + router_identity_id: ROUTER_IDENTITY_ID, + mesh_scope: None, + }, + ) +} + +pub(crate) fn emit_direct_artifact_with_options( + compiled: &CompiledScenario, + options: DirectArtifactBuildOptions<'_>, ) -> Result { let scenario = compiled.scenario(); let endpoint_plan = crate::targets::program_config::build_endpoint_plan(scenario)?; @@ -235,7 +256,7 @@ pub(crate) fn emit_direct_artifact( let route_ports = placeholder_local_route_ports(scenario, &endpoint_plan, &mesh_plan); let mesh_ports_by_component = placeholder_mesh_ports(program_components); - let needs_router = mesh_plan.needs_router() || force_router; + let needs_router = mesh_plan.needs_router() || options.force_router; let router_ports = needs_router.then_some(RouterPorts { mesh: 0, control: 0, @@ -279,11 +300,12 @@ pub(crate) fn emit_direct_artifact( router_ports, addressing: &mesh_addressing, options: MeshConfigBuildOptions { - router_identity_id: ROUTER_IDENTITY_ID, + router_identity_id: options.router_identity_id, + mesh_scope: options.mesh_scope, component_mesh_listen_addr: "127.0.0.1", router_mesh_listen_addr: "127.0.0.1", router_control_listen_addr: "127.0.0.1", - force_router, + force_router: options.force_router, }, })?; @@ -325,7 +347,7 @@ pub(crate) fn emit_direct_artifact( })?; let router_plan = router_ports.map(|ports| DirectRouterPlan { - identity_id: ROUTER_IDENTITY_ID.to_string(), + identity_id: options.router_identity_id.to_string(), mesh_port: ports.mesh, control_port: 0, control_socket_path: DIRECT_CONTROL_SOCKET_RELATIVE_PATH.to_string(), diff --git a/compiler/src/targets/mesh/docker_compose/mod.rs b/compiler/src/targets/mesh/docker_compose/mod.rs index 8985380c..2be9368c 100644 --- a/compiler/src/targets/mesh/docker_compose/mod.rs +++ b/compiler/src/targets/mesh/docker_compose/mod.rs @@ -99,6 +99,29 @@ const LOG_LABEL_SERVICE_NAME: &str = "amber_service_name"; const LOG_LABEL_LIST: &str = "amber_component_moniker,amber_service_name"; pub const COMPOSE_FILENAME: &str = GENERATED_COMPOSE_FILENAME; +fn build_control_socket_init_service(volume_name: &str) -> Service { + let mut service = Service::new("busybox:1.36.1".to_string()); + service.user = Some("0:0".to_string()); + service.command = Some(vec![ + "sh".to_string(), + "-lc".to_string(), + format!( + "mkdir -p {ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER} && chown \ + {ROUTER_RUNTIME_UID}:{ROUTER_RUNTIME_GID} {ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER} \ + && chmod 0700 {ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER}" + ), + ]); + service.volumes.push(format!( + "{volume_name}:{ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER}" + )); + service.restart = Some("no".to_string()); + service +} + +fn component_control_init_service_name(service_name: &str) -> String { + format!("{service_name}-control-init") +} + #[derive(Clone, Copy, Debug, Default)] pub struct DockerComposeReporter; @@ -107,6 +130,13 @@ pub struct DockerComposeArtifact { pub files: BTreeMap, } +#[derive(Clone, Copy, Debug)] +pub(crate) struct DockerComposeArtifactBuildOptions<'a> { + pub(crate) force_router: bool, + pub(crate) router_identity_id: &'a str, + pub(crate) mesh_scope: Option<&'a str>, +} + impl DockerComposeArtifact { pub fn compose_yaml(&self) -> &str { self.files @@ -349,13 +379,34 @@ pub(crate) fn emit_docker_compose_artifact( compiled: &CompiledScenario, force_router: bool, ) -> Result { - render_docker_compose_inner(compiled.scenario(), force_router) - .map_err(DockerComposeError::into_reporter_error) + emit_docker_compose_artifact_with_options( + compiled, + DockerComposeArtifactBuildOptions { + force_router, + router_identity_id: crate::targets::mesh::mesh_config::DEFAULT_ROUTER_ID, + mesh_scope: None, + }, + ) +} + +pub(crate) fn emit_docker_compose_artifact_with_options( + compiled: &CompiledScenario, + options: DockerComposeArtifactBuildOptions<'_>, +) -> Result { + render_docker_compose_inner( + compiled.scenario(), + options.force_router, + options.router_identity_id, + options.mesh_scope, + ) + .map_err(DockerComposeError::into_reporter_error) } fn render_docker_compose_inner( scenario: &Scenario, force_router: bool, + router_identity_id: &str, + mesh_scope: Option<&str>, ) -> DcResult { let transformed = rewrite_framework_docker_as_injected_component(scenario).map_err(dc_other)?; let s = &transformed.scenario; @@ -435,6 +486,8 @@ fn render_docker_compose_inner( router_ports, addressing: &mesh_addressing, options: MeshConfigBuildOptions { + router_identity_id, + mesh_scope, force_router, ..default_mesh_config_build_options() }, @@ -629,22 +682,8 @@ fn render_docker_compose_inner( .entry(ROUTER_CONTROL_SOCKET_VOLUME_NAME.to_string()) .or_insert_with(EmptyMap::default); - let mut control_init_service = Service::new("busybox:1.36.1".to_string()); - control_init_service.user = Some("0:0".to_string()); - control_init_service.command = Some(vec![ - "sh".to_string(), - "-lc".to_string(), - format!( - "mkdir -p {ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER} && chown \ - {ROUTER_RUNTIME_UID}:{ROUTER_RUNTIME_GID} \ - {ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER} && chmod 0700 \ - {ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER}" - ), - ]); - control_init_service.volumes.push(format!( - "{ROUTER_CONTROL_SOCKET_VOLUME_NAME}:{ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER}" - )); - control_init_service.restart = Some("no".to_string()); + let control_init_service = + build_control_socket_init_service(ROUTER_CONTROL_SOCKET_VOLUME_NAME); compose.services.insert( ROUTER_CONTROL_INIT_SERVICE_NAME.to_string(), control_init_service, @@ -742,6 +781,7 @@ fn render_docker_compose_inner( let mut sidecar_env_entries = vec![ format!("AMBER_ROUTER_CONFIG_PATH={}", mesh_config_path()), format!("AMBER_ROUTER_IDENTITY_PATH={}", mesh_identity_path()), + format!("AMBER_ROUTER_CONTROL_SOCKET_PATH={ROUTER_CONTROL_SOCKET_PATH_IN_CONTAINER}"), ]; sidecar_env_entries.extend( mesh_config_plan @@ -765,15 +805,34 @@ fn render_docker_compose_inner( sidecar_service .volumes .push(format!("{sidecar_volume}:{MESH_CONFIG_DIR}:ro")); + let sidecar_control_volume = component_control_socket_volume_expr(&svc.sidecar); + compose + .volumes + .entry(sidecar_control_volume.clone()) + .or_insert_with(EmptyMap::default); + sidecar_service.volumes.push(format!( + "{sidecar_control_volume}:{ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER}" + )); + let sidecar_control_init_service_name = component_control_init_service_name(&svc.sidecar); sidecar_service.depends_on = build_depends_on( false, - vec![( - PROVISIONER_SERVICE_NAME.to_string(), - "service_completed_successfully", - )], + vec![ + ( + PROVISIONER_SERVICE_NAME.to_string(), + "service_completed_successfully", + ), + ( + sidecar_control_init_service_name.clone(), + "service_completed_successfully", + ), + ], ); apply_default_service_hardening(&mut sidecar_service); apply_internal_service_rootfs_hardening(&mut sidecar_service); + compose.services.insert( + sidecar_control_init_service_name, + build_control_socket_init_service(&sidecar_control_volume), + ); compose .services .insert(svc.sidecar.clone(), sidecar_service); @@ -1540,6 +1599,10 @@ fn compose_control_socket_volume_expr() -> String { ) } +fn component_control_socket_volume_expr(service_name: &str) -> String { + format!("{service_name}-control") +} + fn sanitize_service_suffix(s: &str) -> String { let mut out = String::new(); for ch in s.chars() { diff --git a/compiler/src/targets/mesh/docker_compose/tests/rendering.rs b/compiler/src/targets/mesh/docker_compose/tests/rendering.rs index 68a576c8..5dae1ce5 100644 --- a/compiler/src/targets/mesh/docker_compose/tests/rendering.rs +++ b/compiler/src/targets/mesh/docker_compose/tests/rendering.rs @@ -124,6 +124,79 @@ fn compose_storage_volume_names_include_identity_hash() { assert!(child_name.starts_with("amber-storage-root-state-")); } +#[test] +fn compose_sidecars_expose_control_socket_volume() { + let program = lower_test_program( + 0, + json!({ + "image": "alpine:3.20", + "entrypoint": ["sh", "-lc", "sleep infinity"], + "env": {} + }), + ); + let root = Component { + id: ComponentId(0), + parent: None, + moniker: moniker("/"), + digest: digest(0), + config: None, + config_schema: None, + program: Some(program), + slots: BTreeMap::new(), + provides: BTreeMap::new(), + resources: BTreeMap::new(), + metadata: None, + child_templates: BTreeMap::new(), + children: Vec::new(), + }; + let artifact = render_compose(&compile_output(Scenario { + manifest_catalog: BTreeMap::new(), + root: ComponentId(0), + components: vec![Some(root)], + bindings: Vec::new(), + exports: Vec::new(), + })) + .expect("compose render ok"); + let compose = parse_compose(&artifact); + let sidecar_name = compose + .services + .keys() + .find_map(|name| { + let name = name.as_str(); + name.ends_with("-net").then_some(name) + }) + .expect("sidecar service"); + let sidecar = service(&compose, sidecar_name); + assert_eq!( + env_value(sidecar, "AMBER_ROUTER_CONTROL_SOCKET_PATH").as_deref(), + Some("/amber/control/router-control.sock") + ); + assert!( + sidecar + .volumes + .iter() + .any(|mount| mount.ends_with(":/amber/control")), + "sidecar should mount a writable control socket volume: {:?}", + sidecar.volumes + ); + let control_init_name = format!("{sidecar_name}-control-init"); + assert_depends_on( + sidecar, + &control_init_name, + "service_completed_successfully", + ); + let control_init_service = service(&compose, &control_init_name); + assert_eq!(control_init_service.user.as_deref(), Some("0:0")); + assert!( + control_init_service + .volumes + .iter() + .any(|mount| mount.ends_with(":/amber/control")), + "sidecar control init should mount the sidecar control volume: {:?}", + control_init_service.volumes + ); +} + #[test] fn compose_emits_otelcol_agent_and_wires_router_otel_env() { let program = lower_test_program( diff --git a/compiler/src/targets/mesh/internal_images.rs b/compiler/src/targets/mesh/internal_images.rs index b408761b..1b417162 100644 --- a/compiler/src/targets/mesh/internal_images.rs +++ b/compiler/src/targets/mesh/internal_images.rs @@ -1,8 +1,7 @@ -use std::{collections::HashSet, env}; - -use amber_images::{AMBER_DOCKER_GATEWAY, AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER, ImageRef}; - -const DEV_IMAGE_TAGS_ENV: &str = "AMBER_DEV_IMAGE_TAGS"; +use amber_images::{ + AMBER_DOCKER_GATEWAY, AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER, AMBER_SITE_CONTROLLER, + INTERNAL_IMAGE_OVERRIDE_KEYS, ImageRef, override_reference, parse_dev_image_tag_overrides, +}; #[derive(Clone, Debug)] pub(crate) struct InternalImages { @@ -10,6 +9,7 @@ pub(crate) struct InternalImages { pub(crate) provisioner: String, pub(crate) router: String, pub(crate) docker_gateway: String, + pub(crate) site_controller: String, } pub(crate) fn resolve_internal_images() -> Result { @@ -18,70 +18,21 @@ pub(crate) fn resolve_internal_images() -> Result { provisioner: default_reference(&AMBER_PROVISIONER), router: default_reference(&AMBER_ROUTER), docker_gateway: default_reference(&AMBER_DOCKER_GATEWAY), + site_controller: default_reference(&AMBER_SITE_CONTROLLER), }; - let raw = match env::var(DEV_IMAGE_TAGS_ENV) { - Ok(value) => value, - Err(env::VarError::NotPresent) => return Ok(images), - Err(err) => { - return Err(format!("failed to read {DEV_IMAGE_TAGS_ENV}: {err}")); - } - }; - - let raw = raw.trim(); - if raw.is_empty() { - return Err(format!( - "{DEV_IMAGE_TAGS_ENV} is set but empty; expected format \ - \"router=,helper=,provisioner=,docker_gateway=\"" - )); - } - - let mut seen = HashSet::new(); - for entry in raw.split(',') { - let entry = entry.trim(); - if entry.is_empty() { - return Err(format!( - "{DEV_IMAGE_TAGS_ENV} contains an empty entry; expected format \ - \"router=,helper=,provisioner=,docker_gateway=\"" - )); - } - - let mut parts = entry.splitn(2, '='); - let key = parts.next().unwrap().trim(); - let value = parts.next().ok_or_else(|| { - format!("{DEV_IMAGE_TAGS_ENV} entry \"{entry}\" is missing '='; expected key=value") - })?; - let value = value.trim(); - - if key.is_empty() { - return Err(format!( - "{DEV_IMAGE_TAGS_ENV} entry \"{entry}\" is missing a key; expected key=value" - )); - } - if value.is_empty() { - return Err(format!( - "{DEV_IMAGE_TAGS_ENV} entry \"{entry}\" is missing a tag; expected key=tag" - )); - } - if !seen.insert(key.to_string()) { - return Err(format!( - "{DEV_IMAGE_TAGS_ENV} contains duplicate key \"{key}\"" - )); - } - - match key { - "router" => images.router = override_reference(&AMBER_ROUTER, value), - "helper" => images.helper = override_reference(&AMBER_HELPER, value), - "provisioner" => images.provisioner = override_reference(&AMBER_PROVISIONER, value), + for (key, value) in parse_dev_image_tag_overrides(INTERNAL_IMAGE_OVERRIDE_KEYS)? { + match key.as_str() { + "router" => images.router = override_reference(&AMBER_ROUTER, &value), + "helper" => images.helper = override_reference(&AMBER_HELPER, &value), + "provisioner" => images.provisioner = override_reference(&AMBER_PROVISIONER, &value), "docker_gateway" => { - images.docker_gateway = override_reference(&AMBER_DOCKER_GATEWAY, value) + images.docker_gateway = override_reference(&AMBER_DOCKER_GATEWAY, &value) } - _ => { - return Err(format!( - "{DEV_IMAGE_TAGS_ENV} contains unknown key \"{key}\"; expected router, \ - helper, provisioner, docker_gateway" - )); + "site_controller" => { + images.site_controller = override_reference(&AMBER_SITE_CONTROLLER, &value) } + _ => unreachable!("shared image override parser returned an unknown key"), } } @@ -91,7 +42,3 @@ pub(crate) fn resolve_internal_images() -> Result { fn default_reference(image: &ImageRef) -> String { image.reference.to_string() } - -fn override_reference(image: &ImageRef, tag: &str) -> String { - format!("{}/{}:{}", image.registry, image.name, tag) -} diff --git a/compiler/src/targets/mesh/kubernetes/mod.rs b/compiler/src/targets/mesh/kubernetes/mod.rs index 7a4dc1ca..5d5a79e9 100644 --- a/compiler/src/targets/mesh/kubernetes/mod.rs +++ b/compiler/src/targets/mesh/kubernetes/mod.rs @@ -111,6 +111,13 @@ pub struct KubernetesArtifact { pub files: BTreeMap, } +#[derive(Clone, Copy, Debug)] +pub(crate) struct KubernetesArtifactBuildOptions<'a> { + pub(crate) force_router: bool, + pub(crate) router_identity_id: &'a str, + pub(crate) mesh_scope: Option<&'a str>, +} + impl Reporter for KubernetesReporter { type Artifact = KubernetesArtifact; @@ -171,6 +178,20 @@ pub fn render_kubernetes_with_output( pub(crate) fn emit_kubernetes_artifact( compiled: &CompiledScenario, force_router: bool, +) -> KubernetesResult { + emit_kubernetes_artifact_with_options( + compiled, + KubernetesArtifactBuildOptions { + force_router, + router_identity_id: crate::targets::mesh::mesh_config::DEFAULT_ROUTER_ID, + mesh_scope: None, + }, + ) +} + +pub(crate) fn emit_kubernetes_artifact_with_options( + compiled: &CompiledScenario, + options: KubernetesArtifactBuildOptions<'_>, ) -> KubernetesResult { let s = compiled.scenario(); let scenario_digest = @@ -223,7 +244,7 @@ pub(crate) fn emit_kubernetes_artifact( } }); let provisioner_job_name = provisioner_job_name(&scenario_digest); - let needs_router = mesh_plan.needs_router() || force_router; + let needs_router = mesh_plan.needs_router() || options.force_router; let route_ports = allocate_local_route_ports(s, &endpoint_plan, &mesh_plan) .map_err(|e| ReporterError::new(e.to_string()))?; @@ -274,7 +295,9 @@ pub(crate) fn emit_kubernetes_artifact( router_ports, addressing: &mesh_addressing, options: MeshConfigBuildOptions { - force_router, + router_identity_id: options.router_identity_id, + mesh_scope: options.mesh_scope, + force_router: options.force_router, ..default_mesh_config_build_options() }, }) diff --git a/compiler/src/targets/mesh/mesh_config.rs b/compiler/src/targets/mesh/mesh_config.rs index 076402e0..3b8ffad7 100644 --- a/compiler/src/targets/mesh/mesh_config.rs +++ b/compiler/src/targets/mesh/mesh_config.rs @@ -23,6 +23,7 @@ pub(crate) const DEFAULT_ROUTER_ID: &str = "/router"; #[derive(Clone, Copy, Debug)] pub(crate) struct MeshConfigBuildOptions<'a> { pub(crate) router_identity_id: &'a str, + pub(crate) mesh_scope: Option<&'a str>, pub(crate) component_mesh_listen_addr: &'a str, pub(crate) router_mesh_listen_addr: &'a str, pub(crate) router_control_listen_addr: &'a str, @@ -32,6 +33,7 @@ pub(crate) struct MeshConfigBuildOptions<'a> { pub(crate) fn default_mesh_config_build_options() -> MeshConfigBuildOptions<'static> { MeshConfigBuildOptions { router_identity_id: DEFAULT_ROUTER_ID, + mesh_scope: None, component_mesh_listen_addr: "0.0.0.0", router_mesh_listen_addr: "0.0.0.0", router_control_listen_addr: "0.0.0.0", @@ -142,7 +144,10 @@ pub(crate) fn build_mesh_config_plan( return Err(MeshError::new("router ports missing")); } - let mesh_scope = scenario_mesh_scope(scenario)?; + let mesh_scope = match options.mesh_scope { + Some(mesh_scope) => mesh_scope.to_string(), + None => scenario_mesh_scope(scenario)?, + }; let mut identities_by_component: HashMap = HashMap::new(); for &id in mesh_plan.program_components() { @@ -420,6 +425,14 @@ pub(crate) fn build_mesh_config_plan( let mut router_env_passthrough = Vec::new(); let router_config = if needs_router { + push_env_passthrough_once( + &mut router_env_passthrough, + FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, + ); + push_env_passthrough_once( + &mut router_env_passthrough, + FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV, + ); let router_identity = router_identity.expect("router identity should exist"); let router_ports = router_ports.expect("router ports missing"); let router_mesh_port = router_ports.mesh; @@ -497,16 +510,6 @@ pub(crate) fn build_mesh_config_plan( .framework_bindings() .filter(|binding| binding.capability.as_str() == "component") .collect::>(); - if !framework_bindings.is_empty() { - push_env_passthrough_once( - &mut router_env_passthrough, - FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, - ); - push_env_passthrough_once( - &mut router_env_passthrough, - FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV, - ); - } for binding in framework_bindings { let slot_decl = scenario .component(binding.consumer) diff --git a/compiler/src/targets/vm/mod.rs b/compiler/src/targets/vm/mod.rs index 13db0b0f..e0c69e86 100644 --- a/compiler/src/targets/vm/mod.rs +++ b/compiler/src/targets/vm/mod.rs @@ -68,6 +68,13 @@ pub struct VmArtifact { pub files: BTreeMap, } +#[derive(Clone, Copy, Debug)] +pub(crate) struct VmArtifactBuildOptions<'a> { + pub(crate) force_router: bool, + pub(crate) router_identity_id: &'a str, + pub(crate) mesh_scope: Option<&'a str>, +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct VmPlan { pub version: String, @@ -201,6 +208,20 @@ impl Reporter for VmReporter { pub(crate) fn emit_vm_artifact( compiled: &CompiledScenario, force_router: bool, +) -> Result { + emit_vm_artifact_with_options( + compiled, + VmArtifactBuildOptions { + force_router, + router_identity_id: ROUTER_IDENTITY_ID, + mesh_scope: None, + }, + ) +} + +pub(crate) fn emit_vm_artifact_with_options( + compiled: &CompiledScenario, + options: VmArtifactBuildOptions<'_>, ) -> Result { let scenario = compiled.scenario(); let endpoint_plan = crate::targets::program_config::build_endpoint_plan(scenario)?; @@ -248,7 +269,7 @@ pub(crate) fn emit_vm_artifact( let route_ports = placeholder_local_route_ports(scenario, &endpoint_plan, &mesh_plan); let mesh_ports_by_component = placeholder_mesh_ports(program_components); - let needs_router = mesh_plan.needs_router() || force_router; + let needs_router = mesh_plan.needs_router() || options.force_router; let router_ports = needs_router.then_some(RouterPorts { mesh: 0, control: 0, @@ -292,11 +313,12 @@ pub(crate) fn emit_vm_artifact( router_ports, addressing: &mesh_addressing, options: MeshConfigBuildOptions { - router_identity_id: ROUTER_IDENTITY_ID, + router_identity_id: options.router_identity_id, + mesh_scope: options.mesh_scope, component_mesh_listen_addr: "127.0.0.1", router_mesh_listen_addr: "127.0.0.1", router_control_listen_addr: "127.0.0.1", - force_router, + force_router: options.force_router, }, })?; @@ -338,7 +360,7 @@ pub(crate) fn emit_vm_artifact( })?; let router_plan = router_ports.map(|ports| VmRouterPlan { - identity_id: ROUTER_IDENTITY_ID.to_string(), + identity_id: options.router_identity_id.to_string(), mesh_port: ports.mesh, control_port: 0, control_socket_path: DIRECT_CONTROL_SOCKET_RELATIVE_PATH.to_string(), diff --git a/docker/amber-site-controller/Dockerfile b/docker/amber-site-controller/Dockerfile new file mode 100644 index 00000000..eb593480 --- /dev/null +++ b/docker/amber-site-controller/Dockerfile @@ -0,0 +1,84 @@ +# syntax=docker/dockerfile:1.12 + +FROM rust:1.93.1-slim-trixie AS base-builder + +WORKDIR /app + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ + apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + g++ \ + musl-tools \ + pkg-config + +COPY rust-toolchain.toml . +RUN rustup show +ARG TARGETARCH +RUN case "${TARGETARCH}" in \ + amd64) target="x86_64-unknown-linux-musl" ;; \ + arm64) target="aarch64-unknown-linux-musl" ;; \ + *) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ + esac && \ + echo "${target}" > /tmp/rust-target && \ + rustup target add "${target}" + +FROM base-builder AS builder + +COPY Cargo.toml Cargo.lock ./ +COPY cli/Cargo.toml cli/ +COPY compiler/config/Cargo.toml compiler/config/ +COPY compiler/Cargo.toml compiler/ +COPY runtime/docker-gateway/Cargo.toml runtime/docker-gateway/ +COPY runtime/helper/Cargo.toml runtime/helper/ +COPY images/Cargo.toml images/ +COPY manager/Cargo.toml manager/ +COPY compiler/json5/Cargo.toml compiler/json5/ +COPY compiler/manifest/Cargo.toml compiler/manifest/ +COPY runtime/mesh/Cargo.toml runtime/mesh/ +COPY runtime/proxy/Cargo.toml runtime/proxy/ +COPY runtime/router/Cargo.toml runtime/router/ +COPY runtime/site-controller/Cargo.toml runtime/site-controller/ +COPY compiler/resolver/Cargo.toml compiler/resolver/ +COPY compiler/scenario/Cargo.toml compiler/scenario/ +COPY compiler/template/Cargo.toml compiler/template/ +COPY runtime/provisioner/Cargo.toml runtime/provisioner/ + +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ + mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/router/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src runtime/provisioner/src && \ + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ + cargo fetch --locked + +COPY cli ./cli +COPY compiler ./compiler +COPY images ./images +COPY runtime/helper ./runtime/helper +COPY runtime/mesh ./runtime/mesh +COPY runtime/proxy ./runtime/proxy +COPY runtime/router ./runtime/router +COPY runtime/site-controller ./runtime/site-controller +COPY docker/images.json docker/images.json + +ARG BUILD_MODE=release +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,target=/app/target,sharing=locked \ + target=$(cat /tmp/rust-target) && \ + if [ "$BUILD_MODE" = "release" ]; then \ + cargo build --locked --release -p amber-site-controller --target "${target}"; \ + build_dir=release; \ + else \ + cargo build --locked -p amber-site-controller --target "${target}"; \ + build_dir=debug; \ + fi && \ + install -D -m 0755 /app/target/"${target}"/"${build_dir}"/amber-site-controller /out/amber-site-controller + +FROM docker:28-cli AS runtime + +RUN apk add --no-cache kubectl + +COPY --from=builder /out/amber-site-controller /amber-site-controller + +ENTRYPOINT ["/amber-site-controller"] diff --git a/docker/images.json b/docker/images.json index 1b3e487b..b982d2e8 100644 --- a/docker/images.json +++ b/docker/images.json @@ -36,6 +36,12 @@ "context": ".", "dockerfile": "docker/amber-manager/Dockerfile", "version": "v0.1.x" + }, + { + "name": "amber-site-controller", + "context": ".", + "dockerfile": "docker/amber-site-controller/Dockerfile", + "version": "v0.1.x" } ] } diff --git a/examples/observability-debug/README.md b/examples/observability-debug/README.md index e2c79fcd..5a692410 100644 --- a/examples/observability-debug/README.md +++ b/examples/observability-debug/README.md @@ -1,16 +1,17 @@ # Observability Debug Tutorial -This example is for reading a scenario the way a user experiences it. +This example is for debugging an Amber scenario as a user thinks about it: -The scenario has two components and one external upstream: +- components: `client`, `server` +- user-facing edge refs: `/client.server_api -> /server.api`, `/client.ext_api -> external.ext_api`, `public` -- `public -> /server` -- `/client.server_api -> /server.api` -- `/client.ext_api -> external.ext_api` +The dashboard should not force you to think about Amber's internal mesh. Instead, it gives you three resources: -The important output is Amber's interaction trace stream. `amber run`, `amber attach`, and -`amber logs` render that stream for humans, and the same data is persisted for tooling in -`observability/events.ndjson` under the run root. +- `amber..client`: the client program's stdout/stderr +- `amber..server`: the server program's stdout/stderr +- `amber..bindings`: structured request/response telemetry for the scenario's routed edges + +`` is the Compose project name. If you do not set one, it defaults to `default`. ## Prereqs @@ -18,52 +19,55 @@ The important output is Amber's interaction trace stream. `amber run`, `amber at - `amber` CLI on `PATH` - `python3` on the host -## 1) Start the local upstream +## 1) Start the dashboard ```sh -python3 examples/observability-debug/upstream-sse.py --port 38081 +docker rm -f amber-dashboard >/dev/null 2>&1 || true +amber dashboard --detach ``` -## 2) Start Amber +Dashboard UI: `http://127.0.0.1:18888` -Run the scenario in the foreground: +## 2) Compile + start the scenario ```sh -cd examples/observability-debug -amber run . +OUT=/tmp/amber-observability-debug +rm -rf "$OUT" +amber compile examples/observability-debug/scenario.json5 --docker-compose "$OUT" +COMPOSE_PROJECT_NAME=amberdemo docker compose -f "$OUT/compose.yaml" up -d ``` -When Amber prompts for the `ext_api` slot, provide: +If you are validating local framework changes, set `AMBER_DEV_IMAGE_TAGS` when you run `amber compile`. For this example, router-only changes need `router=`. If your changes affect provisioning or generated mesh config metadata, use `router=,provisioner=` so the provisioned sidecar config matches the runtime you are testing. If your changes affect the site controller image used for compose or Kubernetes sites, add `site_controller=` as well. -```text -http://127.0.0.1:38081 -``` +## 3) Start the local upstream (Terminal A) -Amber prints the run id, run root, exported localhost URLs, and then tails the interaction stream. +```sh +python3 examples/observability-debug/upstream-sse.py --port 38081 +``` -If you prefer a managed background run: +## 4) Wire the external slot and export (Terminal B) ```sh -cd examples/observability-debug -amber run . --detach -amber attach +OUT=/tmp/amber-observability-debug +amber proxy "$OUT" \ + --project-name amberdemo \ + --slot ext_api=127.0.0.1:38081 \ + --export public=127.0.0.1:38080 ``` -`amber ps` lists active runs, `amber logs ` replays the persisted interaction story, and -the run root contains `observability/events.ndjson` for machine tooling. +This enables two manifest-level edges: -If you are validating local runtime image changes, set `AMBER_DEV_IMAGE_TAGS` before `amber run`. -For router-only changes, `router=` is usually enough. If you changed provisioning or generated -mesh metadata too, also include `provisioner=`. +- `/client -> ext_api` +- `public -> /server` -## 3) Generate host traffic +Brief `502` or `503` responses are expected until the slot is registered and the upstream is reachable. -In another terminal, use the `public` URL printed by Amber: +## 5) Generate explicit host traffic (Terminal C) ```sh curl -sS \ -H 'x-amber-tutorial: host-export' \ - http://127.0.0.1:/rpc + http://127.0.0.1:38080/rpc ``` Expected response: @@ -72,41 +76,97 @@ Expected response: {"jsonrpc":"2.0","id":"server-static","method":"tools/list","result":{"source":"server","ok":true}} ``` -The `client` also generates traffic in a loop, so the interaction stream should cover all three -manifest-level stories. +The `client` also generates traffic in a loop, so after this call you have all three stories in the dashboard: + +- `public -> /server` +- `/client.server_api -> /server.api` +- `/client.ext_api -> external.ext_api`, including SSE + +## 6) Read the telemetry + +Open `http://127.0.0.1:18888`. + +### Component resources + +`amber.amberdemo.client` shows the client program's own logs, for example: -## 4) Read the interaction story +- `[client] internal rpc -> server` +- `[client] external rpc -> upstream` +- `[client] external sse -> upstream` -Look for these edges: +`amber.amberdemo.server` shows the server program's own logs, for example: + +- `[server] listening on :9000` +- `[server] received GET /rpc x-amber-tutorial=host-export` +- `[server] responded 200 /rpc id=server-static` + +### Edge resource + +`amber.amberdemo.bindings` is the important one. It contains the request/response lifecycle for user-facing edges. + +Look for these edge refs: - `public` - `/client.server_api -> /server.api` - `/client.ext_api -> external.ext_api` -You should see protocol-aware lines such as: +A novice should be able to read the logs as a story. -- `get agent card` -- `list tools` -- `call tool ...` -- `progress update` +Examples you should see: -Useful things to verify: +- `request received from public by /server [headers]` +- `response sent from /server to public: tools/list result (id=server-static) [body]` +- `request sent from /client.server_api to /server.api [headers]` +- `response received by /client.server_api from /server.api: tools/list result (id=server-static) [body]` +- `request sent from /client.ext_api to external.ext_api: tools/call (id=external-1) [body]` +- `response received by /client.ext_api from external.ext_api: notifications/progress response (id=sse-2) [stream event]` -- related events share a trace id -- the renderer uses scenario edges and components instead of mesh/router internals -- request, response, and stream phases are still visible, but secondary to the semantic action +Important details: -If you started the run detached, `amber logs ` shows the same interaction stream from the -persisted `events.ndjson` file. +- the same request/response chain shares a `traceId` +- request and response logs use slot/capability edge refs instead of mesh/router route ids +- protocol-aware fields are extracted when Amber can understand them: JSON-RPC method/id, MCP tool/progress fields, SSE event ids, and so on -## Cleanup +## API checks + +List the resources the tutorial should create: + +```sh +curl -sS http://127.0.0.1:18888/api/telemetry/resources | jq . +``` + +Show the `public` edge story: -For a foreground run, press `Ctrl-C` in the `amber run` terminal. +```sh +curl -sS 'http://127.0.0.1:18888/api/telemetry/logs?resource=amber.amberdemo.bindings&limit=500' \ + | jq -r ' + .data.resourceLogs[]?.scopeLogs[]?.logRecords[]? + | [ + ([.attributes[]? | select(.key == "event") | .value.stringValue][0] // ""), + (.body.stringValue // ""), + (.traceId // "") + ] + | @tsv' \ + | rg 'public|host-export' +``` -For a detached run: +Show the internal client/server edge story: ```sh -amber stop +curl -sS 'http://127.0.0.1:18888/api/telemetry/traces?resource=amber.amberdemo.bindings&limit=200' \ + | jq -r ' + .data.resourceSpans[]?.scopeSpans[]?.spans[]? + | select((.attributes[]? | select(.key == "amber_edge_ref") | .value.stringValue) == "/client.server_api -> /server.api") + | [.traceId, .name, ([.events[]?.name] | join(" | "))] + | @tsv' ``` -Then stop the upstream process if it is still running. +## Cleanup + +```sh +OUT=/tmp/amber-observability-debug +COMPOSE_PROJECT_NAME=amberdemo docker compose -f "$OUT/compose.yaml" down -v +docker rm -f amber-dashboard >/dev/null 2>&1 || true +pkill -f 'upstream-sse.py --port 38081' >/dev/null 2>&1 || true +pkill -f 'amber proxy /tmp/amber-observability-debug' >/dev/null 2>&1 || true +``` diff --git a/images/src/lib.rs b/images/src/lib.rs index f531fea8..46fa81c8 100644 --- a/images/src/lib.rs +++ b/images/src/lib.rs @@ -1,5 +1,19 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + env, +}; + pub mod versioning; +pub const DEV_IMAGE_TAGS_ENV: &str = "AMBER_DEV_IMAGE_TAGS"; +pub const INTERNAL_IMAGE_OVERRIDE_KEYS: &[&str] = &[ + "router", + "helper", + "provisioner", + "docker_gateway", + "site_controller", +]; + #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] pub struct ImageRef { pub name: &'static str, @@ -8,4 +22,126 @@ pub struct ImageRef { pub reference: &'static str, } +pub fn override_reference(image: &ImageRef, tag: &str) -> String { + format!("{}/{}:{}", image.registry, image.name, tag) +} + +pub fn parse_dev_image_tag_overrides( + allowed_keys: &[&str], +) -> Result, String> { + match env::var(DEV_IMAGE_TAGS_ENV) { + Ok(raw) => parse_dev_image_tag_overrides_from_raw(Some(raw.as_str()), allowed_keys), + Err(env::VarError::NotPresent) => Ok(BTreeMap::new()), + Err(err) => Err(format!("failed to read {DEV_IMAGE_TAGS_ENV}: {err}")), + } +} + +pub fn parse_dev_image_tag_overrides_from_raw( + raw: Option<&str>, + allowed_keys: &[&str], +) -> Result, String> { + let Some(raw) = raw else { + return Ok(BTreeMap::new()); + }; + let raw = raw.trim(); + if raw.is_empty() { + return Err(format!( + "{DEV_IMAGE_TAGS_ENV} is set but empty; expected format {}", + expected_override_format(allowed_keys), + )); + } + + let allowed = allowed_keys + .iter() + .map(|key| (*key).to_string()) + .collect::>(); + let mut overrides = BTreeMap::new(); + for entry in raw.split(',') { + let entry = entry.trim(); + if entry.is_empty() { + return Err(format!( + "{DEV_IMAGE_TAGS_ENV} contains an empty entry; expected format {}", + expected_override_format(allowed_keys), + )); + } + + let mut parts = entry.splitn(2, '='); + let key = parts.next().unwrap_or_default().trim(); + let value = parts.next().ok_or_else(|| { + format!("{DEV_IMAGE_TAGS_ENV} entry \"{entry}\" is missing '='; expected key=value") + })?; + let value = value.trim(); + + if key.is_empty() { + return Err(format!( + "{DEV_IMAGE_TAGS_ENV} entry \"{entry}\" is missing a key; expected key=value" + )); + } + if value.is_empty() { + return Err(format!( + "{DEV_IMAGE_TAGS_ENV} entry \"{entry}\" is missing a tag; expected key=tag" + )); + } + if !allowed.contains(key) { + return Err(format!( + "{DEV_IMAGE_TAGS_ENV} contains unknown key \"{key}\"; expected {}", + allowed_keys.join(", "), + )); + } + if overrides + .insert(key.to_string(), value.to_string()) + .is_some() + { + return Err(format!( + "{DEV_IMAGE_TAGS_ENV} contains duplicate key \"{key}\"" + )); + } + } + Ok(overrides) +} + +fn expected_override_format(allowed_keys: &[&str]) -> String { + let entries = allowed_keys + .iter() + .map(|key| format!("{key}=")) + .collect::>() + .join(","); + format!("\"{entries}\"") +} + include!(concat!(env!("OUT_DIR"), "/images.rs")); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_dev_image_tag_overrides_accepts_site_controller_key() { + let overrides = parse_dev_image_tag_overrides_from_raw( + Some("router=dev-router,site_controller=dev-controller"), + INTERNAL_IMAGE_OVERRIDE_KEYS, + ) + .expect("dev image tag overrides should parse"); + + assert_eq!( + overrides.get("router").map(String::as_str), + Some("dev-router") + ); + assert_eq!( + overrides.get("site_controller").map(String::as_str), + Some("dev-controller") + ); + } + + #[test] + fn parse_dev_image_tag_overrides_rejects_unknown_key() { + let err = parse_dev_image_tag_overrides_from_raw( + Some("router=dev-router,unknown=dev"), + INTERNAL_IMAGE_OVERRIDE_KEYS, + ) + .expect_err("unknown override keys should be rejected"); + + assert!(err.contains("unknown key")); + assert!(err.contains("site_controller")); + } +} diff --git a/runtime/proxy/src/tests.rs b/runtime/proxy/src/tests.rs index ce3dc6c2..af9736b0 100644 --- a/runtime/proxy/src/tests.rs +++ b/runtime/proxy/src/tests.rs @@ -471,7 +471,7 @@ fn resolve_control_endpoint_preserves_nested_compose_volume_socket_path() { mesh_port: 24000, control_port: 24100, compose_project: None, - control_socket: Some("/site/compose_local/router-control.sock".to_string()), + control_socket: Some("/router-control.sock".to_string()), control_socket_volume: Some( "${COMPOSE_PROJECT_NAME:-default}_amber-router-control".to_string(), ), @@ -492,7 +492,7 @@ fn resolve_control_endpoint_preserves_nested_compose_volume_socket_path() { panic!("expected compose volume socket endpoint"); }; assert_eq!(volume, "mixed-stack_amber-router-control"); - assert_eq!(socket_path, "/site/compose_local/router-control.sock"); + assert_eq!(socket_path, "/router-control.sock"); } #[test] @@ -505,7 +505,7 @@ fn resolve_control_endpoint_prefers_router_metadata_compose_project() { mesh_port: 24000, control_port: 24100, compose_project: Some("dynamic-stack".to_string()), - control_socket: Some("/site/compose_local/router-control.sock".to_string()), + control_socket: Some("/router-control.sock".to_string()), control_socket_volume: Some( "${COMPOSE_PROJECT_NAME:-default}_amber-router-control".to_string(), ), @@ -525,7 +525,7 @@ fn resolve_control_endpoint_prefers_router_metadata_compose_project() { panic!("expected compose volume socket endpoint"); }; assert_eq!(volume, "dynamic-stack_amber-router-control"); - assert_eq!(socket_path, "/site/compose_local/router-control.sock"); + assert_eq!(socket_path, "/router-control.sock"); } #[test] diff --git a/runtime/site-controller/Cargo.toml b/runtime/site-controller/Cargo.toml index 31da23f4..4feaf417 100644 --- a/runtime/site-controller/Cargo.toml +++ b/runtime/site-controller/Cargo.toml @@ -15,6 +15,7 @@ amber-resolver = { workspace = true } amber-scenario = { workspace = true } axum = { version = "0.8.6", features = ["http1", "json", "tokio"] } base64 = { workspace = true } +libc = "0.2.177" miette = { workspace = true, features = ["fancy"] } reqwest = { workspace = true } regex = "1.11.1" @@ -22,6 +23,8 @@ rmcp = { workspace = true, features = ["server", "macros", "transport-streamable serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } serde_yaml = { workspace = true } +sha2 = "0.10.9" +tar = "0.4.44" tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "signal", "time"] } url = { workspace = true } diff --git a/runtime/site-controller/src/bin/amber-site-controller.rs b/runtime/site-controller/src/bin/amber-site-controller.rs index c9f588da..fbe0d00d 100644 --- a/runtime/site-controller/src/bin/amber-site-controller.rs +++ b/runtime/site-controller/src/bin/amber-site-controller.rs @@ -1,23 +1,10 @@ -use std::{ - env, - os::unix::process::CommandExt as _, - path::{Path, PathBuf}, - process::Command, -}; +use std::{env, path::PathBuf}; -use miette::{IntoDiagnostic as _, Result, WrapErr as _}; +use miette::Result; -fn main() -> Result<()> { - let plan = parse_args()?; - let amber = amber_cli_executable()?; - let err = Command::new(amber) - .arg("run-site-controller") - .arg("--plan") - .arg(plan) - .exec(); - Err(miette::miette!( - "failed to exec amber site controller wrapper: {err}" - )) +#[tokio::main(flavor = "multi_thread")] +async fn main() -> Result<()> { + amber_site_controller::run_site_controller_default(parse_args()?).await } fn parse_args() -> Result { @@ -41,35 +28,3 @@ fn parse_args() -> Result { fn usage_error() -> Result { Err(miette::miette!("usage: amber-site-controller --plan FILE")) } - -fn amber_cli_executable() -> Result { - if let Some(path) = env::var_os("CARGO_BIN_EXE_amber") { - let path = PathBuf::from(path); - if path.is_file() { - return Ok(path); - } - } - - let current = env::current_exe() - .into_diagnostic() - .wrap_err("failed to resolve amber executable path")?; - if let Some(candidate) = sibling_binary(¤t, "amber") { - return Ok(candidate); - } - Err(miette::miette!( - "failed to locate the amber executable next to {}", - current.display() - )) -} - -fn sibling_binary(current: &Path, name: &str) -> Option { - let exe_name = format!("{name}{}", std::env::consts::EXE_SUFFIX); - let bin_dir = current.parent()?; - for dir in [Some(bin_dir), bin_dir.parent()].into_iter().flatten() { - let candidate = dir.join(&exe_name); - if candidate.is_file() { - return Some(candidate); - } - } - None -} diff --git a/runtime/site-controller/src/control_state_api.rs b/runtime/site-controller/src/control_state_api.rs index 109b4d05..cd3b174a 100644 --- a/runtime/site-controller/src/control_state_api.rs +++ b/runtime/site-controller/src/control_state_api.rs @@ -3,11 +3,11 @@ use base64::Engine as _; use super::{ dynamic_caps, orchestration::{ - ProtocolApiError, dynamic_capability_component_runtime_endpoint, load_site_manager_state, - load_site_runtime_plan, publish_dynamic_capability_origin, + ProtocolApiError, load_site_manager_state, local_router_identity_for_overlay, + publish_dynamic_capability_origin, }, - planner::{ControlStateApp, protocol_error}, - state::persist_control_state_update, + planner::{ControlStateApp, live_assignment_map, protocol_error}, + state::{FrameworkControlState, persist_control_state_update}, *, }; @@ -157,9 +157,56 @@ pub(crate) async fn execute_dynamic_caps_mutate( }) } -async fn resolve_dynamic_capability_origin( +pub(crate) fn local_component_runtime( app: &ControlStateApp, - request: dynamic_caps::ControlDynamicResolveOriginRequest, + state: &FrameworkControlState, + logical_component_id: &str, +) -> std::result::Result { + let moniker = dynamic_caps::moniker_from_logical_component_id(logical_component_id)?; + let assignments = live_assignment_map(state); + let site_id = assignments.get(moniker).ok_or_else(|| { + protocol_error( + ProtocolErrorCode::OriginUnavailable, + &format!("live component `{logical_component_id}` is not assigned to a live site"), + ) + })?; + if site_id != &app.controller_plan.site_id { + return Err(protocol_error( + ProtocolErrorCode::OriginUnavailable, + &format!( + "live component `{logical_component_id}` is assigned to site `{site_id}`, not \ + local site `{}`", + app.controller_plan.site_id + ), + )); + } + let site_plan = site_controller_runtime_plan_from_controller_plan(&app.controller_plan); + app.runtime + .collect_live_component_runtime_metadata(&site_plan) + .map_err(|err| { + protocol_error( + ProtocolErrorCode::OriginUnavailable, + &format!( + "failed to resolve live runtime metadata for component \ + `{logical_component_id}` on site `{site_id}`: {err}" + ), + ) + })? + .remove(moniker) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::OriginUnavailable, + &format!( + "live runtime metadata for component `{logical_component_id}` is unavailable \ + on site `{site_id}`" + ), + ) + }) +} + +pub(crate) async fn resolve_dynamic_capability_origin_internal( + app: &ControlStateApp, + request: dynamic_caps::InternalDynamicResolveOriginRequest, ) -> std::result::Result { let state = app.control_state.lock().await.clone(); let source_key = dynamic_caps::source_key_from_control_request(&request.source); @@ -179,38 +226,9 @@ async fn resolve_dynamic_capability_origin( "dynamic capability root authority is no longer live", )) })?; - let holder_runtime = - dynamic_capability_component_runtime_endpoint(app, &state, &request.holder_component_id)?; - let holder_plan = load_site_runtime_plan(app, &holder_runtime.site_id)?; - let origin_runtime = - dynamic_capability_component_runtime_endpoint(app, &state, &root.holder_component_id)?; - let origin_site_id = origin_runtime.site_id.clone(); - let origin_plan = load_site_runtime_plan(app, &origin_site_id)?; - let origin_manager_state = load_site_manager_state(app, &origin_site_id)?; - let origin_peer_id = origin_manager_state - .router_identity_id - .clone() - .ok_or_else(|| { - ProtocolApiError::from(protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "site `{origin_site_id}` does not expose a live router identity for dynamic \ - capability publication" - ), - )) - })?; - let origin_peer_key_b64 = origin_manager_state - .router_public_key_b64 - .clone() - .ok_or_else(|| { - ProtocolApiError::from(protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "site `{origin_site_id}` does not expose a live router public key for dynamic \ - capability publication" - ), - )) - })?; + let _origin_runtime = local_component_runtime(app, &state, &root.holder_component_id)?; + let origin_manager_state = load_site_manager_state(app, &app.controller_plan.site_id)?; + let origin_peer = local_router_identity_for_overlay(app).await?; let origin_peer_addr = origin_manager_state .router_mesh_addr .as_deref() @@ -218,20 +236,26 @@ async fn resolve_dynamic_capability_origin( ProtocolApiError::from(protocol_error( ProtocolErrorCode::OriginUnavailable, &format!( - "site `{origin_site_id}` does not expose a live router mesh address for \ - dynamic capability publication" + "site `{}` does not expose a live router mesh address for dynamic capability \ + publication", + app.controller_plan.site_id ), )) }) .and_then(|router_mesh_addr| { app.runtime - .router_mesh_addr_for_consumer(origin_plan.kind, holder_plan.kind, router_mesh_addr) + .router_mesh_addr_for_consumer( + app.controller_plan.kind, + request.holder_site_kind, + router_mesh_addr, + ) .map_err(|err| { ProtocolApiError::from(protocol_error( ProtocolErrorCode::OriginUnavailable, &format!( - "site `{origin_site_id}` exposes an invalid live router mesh address \ - for dynamic capability publication: {err}" + "site `{}` exposes an invalid live router mesh address for dynamic \ + capability publication: {err}", + app.controller_plan.site_id ), )) }) @@ -246,15 +270,14 @@ async fn resolve_dynamic_capability_origin( ); let publish = publish_dynamic_capability_origin( app, - &origin_site_id, + &app.controller_plan.site_id, &dynamic_caps::PublishDynamicCapabilityOriginRequest { overlay_id, route_id: route_id.clone(), root_authority_selector: resolved_source.root_authority_selector.clone(), allowed_peers: vec![dynamic_caps::DynamicCapabilityAllowedPeer { - peer_id: holder_runtime.runtime.mesh_config.identity.id.clone(), - peer_key_b64: base64::engine::general_purpose::STANDARD - .encode(holder_runtime.runtime.mesh_config.identity.public_key), + peer_id: request.holder_peer_id.clone(), + peer_key_b64: request.holder_peer_key_b64.clone(), }], }, ) @@ -273,8 +296,31 @@ async fn resolve_dynamic_capability_origin( origin_route_id: publish.route_id, origin_capability: publish.capability, origin_protocol: publish.protocol, - origin_peer_id, - origin_peer_key_b64, + origin_peer_id: origin_peer.id, + origin_peer_key_b64: base64::engine::general_purpose::STANDARD + .encode(origin_peer.public_key), origin_peer_addr, }) } + +async fn resolve_dynamic_capability_origin( + app: &ControlStateApp, + request: dynamic_caps::ControlDynamicResolveOriginRequest, +) -> std::result::Result { + let holder_runtime = { + let state = app.control_state.lock().await.clone(); + local_component_runtime(app, &state, &request.holder_component_id)? + }; + resolve_dynamic_capability_origin_internal( + app, + dynamic_caps::InternalDynamicResolveOriginRequest { + holder_component_id: request.holder_component_id, + source: request.source, + holder_peer_id: holder_runtime.mesh_config.identity.id.clone(), + holder_peer_key_b64: base64::engine::general_purpose::STANDARD + .encode(holder_runtime.mesh_config.identity.public_key), + holder_site_kind: app.controller_plan.kind, + }, + ) + .await +} diff --git a/runtime/site-controller/src/default_runtime.rs b/runtime/site-controller/src/default_runtime.rs new file mode 100644 index 00000000..6bae58ed --- /dev/null +++ b/runtime/site-controller/src/default_runtime.rs @@ -0,0 +1,1907 @@ +#[cfg(target_os = "linux")] +use std::net::Ipv4Addr; +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + env, fs, + io::{Read as _, Write as _}, + net::{SocketAddr, TcpListener}, + path::{Path, PathBuf}, + process::{Child, Command, Stdio}, + sync::{Arc, OnceLock}, + time::Duration, +}; + +use amber_compiler::{ + mesh::ProxyMetadata, + reporter::{ + direct::{DirectPlan, DirectRuntimeUrlSource}, + vm::VmPlan, + }, + run_plan::{RunLink, SiteKind}, +}; +use amber_manifest::NetworkProtocol; +use amber_mesh::{ + HttpRoutePlugin, InboundRoute, InboundTarget, MESH_CONFIG_FILENAME, MESH_IDENTITY_FILENAME, + MeshConfigPublic, MeshIdentity, MeshIdentityPublic, MeshIdentitySecret, MeshPeer, + MeshPeerTemplate, MeshProtocol, MeshProvisionOutput, MeshProvisionPlan, MeshProvisionTarget, + MeshProvisionTargetKind, OutboundRoute, component_route_id, router_dynamic_export_route_id, + router_export_route_id, +}; +use amber_proxy::{ + ControlEndpoint, apply_route_overlay_with_retry, load_output_proxy_metadata, + revoke_route_overlay_with_retry, +}; +use miette::{Context as _, IntoDiagnostic as _, Result}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map as JsonMap, Value as JsonValue}; +use sha2::{Digest as _, Sha256}; +use tokio::{ + io::{AsyncReadExt as _, AsyncWriteExt as _}, + sync::Mutex as AsyncMutex, + time::{Instant, sleep}, +}; +use url::Url; + +mod child_runtime; +mod compose_controller; +mod dynamic_routes; +mod kubernetes_controller; +mod site_artifacts; +mod site_runtime_support; + +pub(crate) use self::child_runtime::default_site_controller_runtime; +pub(super) use self::site_artifacts::*; +pub use self::{ + child_runtime::cleanup_dynamic_site_children, + compose_controller::inject_compose_site_controller, + kubernetes_controller::inject_kubernetes_site_controller, + site_artifacts::{ + SiteControllerPeerRouterRoute, add_compose_router_published_route_ports, + inject_site_controller_peer_router_routes, set_compose_router_published_mesh_port, + set_site_artifact_mesh_identity_seed, + }, + site_runtime_support::{ + host_service_bind_addr_for_consumer, observability_endpoint_for_site, + prepare_kubernetes_artifact_namespace, router_mesh_addr_for_consumer, walk_files, + }, +}; +use self::{ + child_runtime::{ + BridgeProxyHandle, BridgeProxyKey, DynamicComposeChildMetadata, SiteControllerRuntimeApp, + SiteControllerRuntimeChildRecord, SiteControllerRuntimeState, StoredRouteOverlayPayload, + dynamic_compose_child_metadata_path, dynamic_route_overlay_path, + load_dynamic_proxy_exports_metadata, site_controller_runtime_child_runtime_root, + site_controller_runtime_state_path, write_dynamic_route_overlay_payload, + }, + dynamic_routes::*, + site_runtime_support::*, +}; +use super::*; +use crate::{ + http::{read_json, write_json}, + planner::{ + LocalChildRuntimeSpec, build_desired_site_artifact_files, build_local_child_runtime_spec, + child_runtime_site_id, + }, + runtime_api::{SharedSiteControllerRuntime, SiteControllerRuntime}, + state::{FrameworkControlState, LiveChildRecord}, +}; + +const DYNAMIC_COMPOSE_CHILD_SCHEMA: &str = "amber.run.dynamic_compose_child"; +const DYNAMIC_COMPOSE_CHILD_VERSION: u32 = 1; +const DYNAMIC_COMPOSE_MESH_ROOT: &str = ".amber/mesh"; +const DYNAMIC_ROUTE_OVERLAY_FILENAME: &str = "site-router-overlay.json"; +const DYNAMIC_PROXY_EXPORTS_FILENAME: &str = "proxy-exports.json"; +const COMPONENT_CONTROL_SOCKET_PATH_IN_CONTAINER: &str = "/amber/control/router-control.sock"; +const COMPOSE_PROVISIONER_SERVICE_NAME: &str = "amber-provisioner"; +const COMPOSE_ROUTER_SERVICE_NAME: &str = "amber-router"; +const KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH: &str = "01-configmaps/amber-mesh-provision.yaml"; +const KUBERNETES_PROVISIONER_JOB_PATH: &str = "02-rbac/amber-provisioner-job.yaml"; +const KUBERNETES_PROVISIONER_ROLE_PATH: &str = "02-rbac/amber-provisioner-role.yaml"; +const KUBERNETES_PROVISIONER_ROLEBINDING_PATH: &str = "02-rbac/amber-provisioner-rolebinding.yaml"; +const KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH: &str = "02-rbac/amber-provisioner-sa.yaml"; +const KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME: &str = "amber-router-external"; +const KUBERNETES_ROUTER_COMPONENT_NAME: &str = "amber-router"; +const SITE_PLAN_SCHEMA: &str = "amber.run.site_supervisor_plan"; +const SITE_PLAN_VERSION: u32 = 1; +const SITE_CONTROLLER_RUNTIME_STATE_SCHEMA: &str = "amber.run.site_controller_runtime_state"; +const SITE_CONTROLLER_RUNTIME_STATE_VERSION: u32 = 1; +const DESIRED_LINKS_SCHEMA: &str = "amber.run.desired_links"; +const DESIRED_LINKS_VERSION: u32 = 1; +const DEFAULT_EXTERNAL_ENV_FILE: &str = "router-external.env"; +const DEFAULT_K8S_OTEL_UPSTREAM: &str = "http://host.docker.internal:18890"; +const CONTAINER_HOST_ALIAS: &str = "host.docker.internal"; +const PROCESS_SHUTDOWN_GRACE_PERIOD: Duration = Duration::from_secs(10); +const KUBERNETES_WORKLOAD_READY_TIMEOUT: Duration = Duration::from_secs(120); +const KUBERNETES_SITE_READY_BUFFER: Duration = Duration::from_secs(30); +const COMPOSE_EGRESS_SUBNET_COUNT: u32 = 1 << 18; + +static KUBERNETES_CONTAINER_HOST_IP: OnceLock> = OnceLock::new(); + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum SiteLifecycleStatus { + Starting, + Running, + Stopping, + Stopped, + Failed, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +struct SiteManagerState { + schema: String, + version: u32, + run_id: String, + site_id: String, + kind: SiteKind, + status: SiteLifecycleStatus, + artifact_dir: String, + supervisor_pid: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + process_pid: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + compose_project: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + kubernetes_namespace: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + port_forward_pid: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + router_control: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + router_identity_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + router_public_key_b64: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + site_controller_pid: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + site_controller_url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + last_error: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct SiteSupervisorPlan { + schema: String, + version: u32, + run_id: String, + mesh_scope: String, + run_root: String, + coordinator_pid: u32, + site_id: String, + kind: SiteKind, + artifact_dir: String, + site_state_root: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + storage_root: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + runtime_root: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + router_mesh_port: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + compose_project: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + kubernetes_namespace: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + port_forward_mesh_port: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + port_forward_control_port: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + observability_endpoint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + site_controller_plan_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + site_controller_url: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + controller_route_ports: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + launch_env: BTreeMap, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +struct DesiredLinkState { + schema: String, + version: u32, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + external_slots: BTreeMap, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + export_peers: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + external_slot_overlays: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + export_peer_overlays: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +struct DesiredExportPeer { + export_name: String, + peer_id: String, + peer_key_b64: String, + protocol: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + route_id: Option, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +struct DirectRuntimeState { + #[serde(default)] + slot_ports_by_component: BTreeMap>, + #[serde(default)] + slot_route_ports_by_component: BTreeMap>>, + #[serde(default)] + dynamic_caps_port_by_component: BTreeMap, + #[serde(default)] + component_mesh_port_by_id: BTreeMap, + #[serde(default)] + router_mesh_port: Option, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +struct VmRuntimeState { + #[serde(default)] + slot_ports_by_component: BTreeMap>, + #[serde(default)] + slot_route_ports_by_component: BTreeMap>>, + #[serde(default)] + route_host_ports_by_component: BTreeMap>>, + #[serde(default)] + endpoint_forwards_by_component: BTreeMap>, + #[serde(default)] + component_mesh_port_by_id: BTreeMap, + #[serde(default)] + router_mesh_port: Option, +} + +fn required_str<'a>(value: Option<&'a str>, label: &str) -> Result<&'a str> { + value.ok_or_else(|| miette::miette!("missing {label}")) +} + +fn direct_runtime_state_path(plan_root: &Path) -> PathBuf { + plan_root.join(".amber").join("direct-runtime.json") +} + +fn direct_current_control_socket_path(plan_root: &Path) -> PathBuf { + amber_mesh::stable_temp_socket_path("amber-direct-control", "current", plan_root) +} + +#[cfg(unix)] +fn ensure_direct_control_socket_link(link: &Path, target: &Path, description: &str) -> Result<()> { + if fs::read_link(link) + .ok() + .is_some_and(|existing_target| existing_target == target) + { + return Ok(()); + } + if fs::symlink_metadata(link).is_ok() { + fs::remove_file(link) + .into_diagnostic() + .wrap_err_with(|| format!("failed to remove stale {description} {}", link.display()))?; + } + std::os::unix::fs::symlink(target, link) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to create {description} {} -> {}", + link.display(), + target.display() + ) + }) +} + +#[cfg(not(unix))] +fn ensure_direct_control_socket_link(link: &Path, target: &Path, description: &str) -> Result<()> { + let _ = (link, target, description); + Err(miette::miette!( + "direct runtime control sockets require unix symlink support" + )) +} + +fn vm_current_control_socket_path(plan_root: &Path) -> PathBuf { + amber_mesh::stable_temp_socket_path("amber-vm-control", "current", plan_root) +} + +#[cfg(unix)] +fn ensure_control_socket_link(link: &Path, target: &Path, description: &str) -> Result<()> { + if fs::read_link(link) + .ok() + .is_some_and(|existing_target| existing_target == target) + { + return Ok(()); + } + if fs::symlink_metadata(link).is_ok() { + fs::remove_file(link) + .into_diagnostic() + .wrap_err_with(|| format!("failed to remove stale {description} {}", link.display()))?; + } + std::os::unix::fs::symlink(target, link) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to create {description} {} -> {}", + link.display(), + target.display() + ) + }) +} + +#[cfg(not(unix))] +fn ensure_control_socket_link(link: &Path, target: &Path, description: &str) -> Result<()> { + let _ = (link, target, description); + Err(miette::miette!( + "vm runtime control sockets require unix symlink support" + )) +} + +fn write_vm_runtime_state(plan_root: &Path, state: &VmRuntimeState) -> Result<()> { + let path = plan_root.join(".amber").join("vm-runtime.json"); + let parent = path + .parent() + .ok_or_else(|| miette::miette!("invalid vm runtime state path"))?; + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create vm runtime dir {}", parent.display()))?; + write_json(&path, state) +} + +const TCG_VM_STARTUP_TIMEOUT: Duration = Duration::from_secs(720); + +fn vm_endpoint_forward_ready_timeout() -> Duration { + if cfg!(target_os = "macos") && env::var_os("AMBER_VM_FORCE_TCG").is_some() { + TCG_VM_STARTUP_TIMEOUT + } else { + Duration::from_secs(120) + } +} + +fn required_existing_mesh_peer_identities( + plan: &MeshProvisionPlan, + available_peer_identities_by_id: &BTreeMap, +) -> Result> { + let target_ids = plan + .targets + .iter() + .map(|target| target.config.identity.id.as_str()) + .collect::>(); + let required_peer_ids = plan + .targets + .iter() + .flat_map(|target| target.config.peers.iter()) + .filter(|peer| !target_ids.contains(peer.id.as_str())) + .map(|peer| peer.id.as_str()) + .collect::>(); + + required_peer_ids + .into_iter() + .map(|peer_id| { + let identity = available_peer_identities_by_id + .get(peer_id) + .cloned() + .ok_or_else(|| { + miette::miette!( + "mesh provision plan requires existing peer identity {peer_id}, but it is \ + not currently available" + ) + })?; + Ok((peer_id.to_string(), identity)) + }) + .collect() +} + +fn mesh_output_dir_for_target(root: &Path, target: &MeshProvisionTarget) -> Result { + match &target.output { + MeshProvisionOutput::Filesystem { dir } => { + let path = Path::new(dir); + if path.is_absolute() { + return Err(miette::miette!( + "mesh provision plan contains absolute filesystem output path {}", + path.display() + )); + } + Ok(root.join(path)) + } + MeshProvisionOutput::KubernetesSecret { name, .. } => Err(miette::miette!( + "local runtime does not support kubernetes provision target {name}" + )), + } +} + +fn provision_mesh_filesystem_with_peer_identities( + plan: &MeshProvisionPlan, + root: &Path, + existing_peer_identities_by_id: &BTreeMap, +) -> Result<()> { + let mut identities: HashMap = HashMap::new(); + for identity in existing_peer_identities_by_id.values() { + identities.insert( + identity.id.clone(), + MeshIdentity { + id: identity.id.clone(), + public_key: identity.public_key, + private_key: [0; 64], + mesh_scope: identity.mesh_scope.clone(), + }, + ); + } + for target in &plan.targets { + let id = target.config.identity.id.clone(); + let mesh_scope = target.config.identity.mesh_scope.clone(); + identities + .entry(id) + .or_insert_with(|| match plan.identity_seed.as_deref() { + Some(seed) => { + MeshIdentity::derive(target.config.identity.id.clone(), mesh_scope, seed) + } + None => MeshIdentity::generate(target.config.identity.id.clone(), mesh_scope), + }); + } + for target in &plan.targets { + let output_dir = mesh_output_dir_for_target(root, target)?; + fs::create_dir_all(&output_dir) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to create mesh output directory {}", + output_dir.display() + ) + })?; + let identity = identities + .get(&target.config.identity.id) + .ok_or_else(|| { + miette::miette!( + "missing generated identity for {}", + target.config.identity.id + ) + })? + .clone(); + let identity_secret = MeshIdentitySecret::from_identity(&identity); + let public_config = target.config.to_public_config(&identities).map_err(|err| { + miette::miette!( + "failed to render mesh config for {}: {err}", + target.config.identity.id + ) + })?; + write_json(&output_dir.join(MESH_IDENTITY_FILENAME), &identity_secret)?; + write_json(&output_dir.join(MESH_CONFIG_FILENAME), &public_config)?; + } + Ok(()) +} + +fn project_existing_peer_identities_into_mesh_config( + path: &Path, + existing_peer_identities_by_id: &BTreeMap, +) -> Result<()> { + if existing_peer_identities_by_id.is_empty() || !path.is_file() { + return Ok(()); + } + let mut config: MeshConfigPublic = read_json(path, "mesh config")?; + let canonical_mesh_scope = existing_peer_identities_by_id + .values() + .find_map(|identity| identity.mesh_scope.clone()); + let mut changed = false; + if let Some(identity) = existing_peer_identities_by_id.get(&config.identity.id) { + if config.identity.public_key != identity.public_key + || config.identity.mesh_scope != identity.mesh_scope + { + config.identity.public_key = identity.public_key; + config.identity.mesh_scope = identity.mesh_scope.clone(); + changed = true; + } + } else if let Some(mesh_scope) = canonical_mesh_scope.as_ref() + && config.identity.mesh_scope.as_deref() != Some(mesh_scope.as_str()) + { + config.identity.mesh_scope = Some(mesh_scope.clone()); + changed = true; + } + for peer in &mut config.peers { + let Some(identity) = existing_peer_identities_by_id.get(&peer.id) else { + continue; + }; + if peer.public_key != identity.public_key { + peer.public_key = identity.public_key; + changed = true; + } + } + if changed { + write_json(path, &config)?; + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn rewrite_peer_addr_for_slirp_gateway(peer_addr: &str) -> String { + let Ok(addr) = peer_addr.parse::() else { + return peer_addr.to_string(); + }; + if !addr.ip().is_loopback() { + return peer_addr.to_string(); + } + SocketAddr::from((Ipv4Addr::new(10, 0, 2, 2), addr.port())).to_string() +} + +#[derive(Debug, Deserialize)] +struct KubernetesSecretPayload { + #[serde(default)] + data: BTreeMap, +} +fn rewrite_dynamic_proxy_metadata( + artifact_root: &Path, + proxy_exports: &BTreeMap, + kind: SiteKind, +) -> Result<()> { + if proxy_exports.is_empty() { + return Ok(()); + } + if kind == SiteKind::Compose { + let path = artifact_root.join("compose.yaml"); + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid compose file {}", path.display()))?; + let Some(root) = document.as_mapping_mut() else { + return Err(miette::miette!( + "compose file {} is not a YAML mapping", + path.display() + )); + }; + let x_amber_key = serde_yaml::Value::String("x-amber".to_string()); + let Some(x_amber) = root.get_mut(&x_amber_key) else { + return Err(miette::miette!( + "compose file {} is missing x-amber metadata", + path.display() + )); + }; + let exports = proxy_exports + .iter() + .map(|(name, export)| { + Ok(( + serde_yaml::Value::String(name.clone()), + serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([ + ( + serde_yaml::Value::String("component".to_string()), + serde_yaml::Value::String(export.component.clone()), + ), + ( + serde_yaml::Value::String("provide".to_string()), + serde_yaml::Value::String(export.provide.clone()), + ), + ( + serde_yaml::Value::String("protocol".to_string()), + serde_yaml::Value::String(export.protocol.clone()), + ), + ( + serde_yaml::Value::String("router_mesh_port".to_string()), + serde_yaml::Value::Number(0u64.into()), + ), + ( + serde_yaml::Value::String("route_id".to_string()), + serde_yaml::Value::String(dynamic_proxy_export_route_id(name, export)?), + ), + ])), + )) + }) + .collect::>()?; + let mut metadata = x_amber.as_mapping().cloned().ok_or_else(|| { + miette::miette!( + "compose file {} has non-mapping x-amber metadata", + path.display() + ) + })?; + metadata.insert( + serde_yaml::Value::String("exports".to_string()), + serde_yaml::Value::Mapping(exports), + ); + *x_amber = serde_yaml::Value::Mapping(metadata); + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(&path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) + } else { + let exports = proxy_exports + .iter() + .map(|(name, export)| { + Ok(( + name.clone(), + JsonValue::Object(JsonMap::from_iter([ + ( + "component".to_string(), + JsonValue::String(export.component.clone()), + ), + ( + "provide".to_string(), + JsonValue::String(export.provide.clone()), + ), + ( + "protocol".to_string(), + JsonValue::String(export.protocol.clone()), + ), + ("router_mesh_port".to_string(), JsonValue::from(0u64)), + ( + "route_id".to_string(), + JsonValue::String(dynamic_proxy_export_route_id(name, export)?), + ), + ])), + )) + }) + .collect::>>()?; + let path = artifact_root.join("amber-proxy.json"); + let mut metadata: JsonValue = read_json(&path, "proxy metadata")?; + let Some(object) = metadata.as_object_mut() else { + return Err(miette::miette!( + "proxy metadata {} is not a JSON object", + path.display() + )); + }; + object.insert("exports".to_string(), JsonValue::Object(exports)); + write_json(&path, &metadata) + } +} + +fn rewrite_dynamic_compose_proxy_metadata( + artifact_root: &Path, + compose_project: &str, +) -> Result<()> { + let path = artifact_root.join("compose.yaml"); + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid compose file {}", path.display()))?; + let Some(root) = document.as_mapping_mut() else { + return Err(miette::miette!( + "compose file {} is not a YAML mapping", + path.display() + )); + }; + let x_amber_key = serde_yaml::Value::String("x-amber".to_string()); + let Some(x_amber) = root.get_mut(&x_amber_key) else { + return Err(miette::miette!( + "compose file {} is missing x-amber metadata", + path.display() + )); + }; + let mut metadata: ProxyMetadata = serde_yaml::from_value(x_amber.clone()) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "compose file {} has invalid x-amber metadata", + path.display() + ) + })?; + if let Some(router) = metadata.router.as_mut() { + router.compose_project = Some(compose_project.to_string()); + } + *x_amber = serde_yaml::to_value(&metadata) + .into_diagnostic() + .wrap_err("failed to serialize compose proxy metadata")?; + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(&path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +pub(super) fn yaml_string(value: &str) -> serde_yaml::Value { + serde_yaml::Value::String(value.to_string()) +} + +fn compose_services<'a>( + document: &'a serde_yaml::Value, + path: &Path, +) -> Result<&'a serde_yaml::Mapping> { + document + .as_mapping() + .and_then(|root| root.get(yaml_string("services"))) + .and_then(serde_yaml::Value::as_mapping) + .ok_or_else(|| { + miette::miette!( + "compose file {} is missing a services mapping", + path.display() + ) + }) +} + +pub(super) fn compose_services_mut<'a>( + document: &'a mut serde_yaml::Value, + path: &Path, +) -> Result<&'a mut serde_yaml::Mapping> { + document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("services"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or_else(|| { + miette::miette!( + "compose file {} is missing a services mapping", + path.display() + ) + }) +} + +fn compose_networks_mut<'a>( + document: &'a mut serde_yaml::Value, + path: &Path, +) -> Result> { + let Some(root) = document.as_mapping_mut() else { + return Err(miette::miette!( + "compose file {} is not a mapping", + path.display() + )); + }; + Ok(root + .get_mut(yaml_string("networks")) + .and_then(serde_yaml::Value::as_mapping_mut)) +} + +pub fn assign_compose_egress_network_subnets( + artifact_dir: &Path, + run_id: &str, + site_id: &str, +) -> Result<()> { + let compose_path = artifact_dir.join("compose.yaml"); + if !compose_path.is_file() { + return Ok(()); + } + + let mut document = read_compose_document(&compose_path)?; + let Some(networks) = compose_networks_mut(&mut document, &compose_path)? else { + return Ok(()); + }; + + let mut used_subnets = networks + .values() + .filter_map(compose_network_subnet) + .map(str::to_string) + .collect::>(); + let mut changed = false; + + for (name, network) in networks.iter_mut() { + let Some(name) = name.as_str() else { + continue; + }; + if !name.starts_with("amber_egress_") || compose_network_subnet(network).is_some() { + continue; + } + let subnet = next_compose_egress_subnet(run_id, site_id, name, &used_subnets); + set_compose_network_subnet(network, &subnet)?; + used_subnets.insert(subnet); + changed = true; + } + + if !changed { + return Ok(()); + } + + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", compose_path.display()))?; + fs::write(&compose_path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", compose_path.display())) +} + +fn compose_network_subnet(network: &serde_yaml::Value) -> Option<&str> { + network + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("ipam"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|ipam| ipam.get(yaml_string("config"))) + .and_then(serde_yaml::Value::as_sequence) + .and_then(|configs| configs.first()) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|config| config.get(yaml_string("subnet"))) + .and_then(serde_yaml::Value::as_str) +} + +fn set_compose_network_subnet(network: &mut serde_yaml::Value, subnet: &str) -> Result<()> { + let Some(mapping) = network.as_mapping_mut() else { + return Err(miette::miette!( + "compose network definition is not a mapping" + )); + }; + mapping.insert( + yaml_string("ipam"), + serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( + yaml_string("config"), + serde_yaml::Value::Sequence(vec![serde_yaml::Value::Mapping( + serde_yaml::Mapping::from_iter([(yaml_string("subnet"), yaml_string(subnet))]), + )]), + )])), + ); + Ok(()) +} + +fn next_compose_egress_subnet( + run_id: &str, + site_id: &str, + network_name: &str, + used_subnets: &BTreeSet, +) -> String { + let digest = Sha256::digest(format!("{run_id}:{site_id}:{network_name}")); + let mut bytes = [0u8; 4]; + bytes.copy_from_slice(&digest[..4]); + let base = u32::from_be_bytes(bytes) % COMPOSE_EGRESS_SUBNET_COUNT; + for offset in 0..COMPOSE_EGRESS_SUBNET_COUNT { + let candidate = + compose_egress_subnet_from_index((base + offset) % COMPOSE_EGRESS_SUBNET_COUNT); + if !used_subnets.contains(&candidate) { + return candidate; + } + } + unreachable!("compose egress subnet pool exhausted") +} + +fn compose_egress_subnet_from_index(index: u32) -> String { + let second_octet = 64 + ((index >> 12) & 0x3f); + let third_octet = (index >> 4) & 0xff; + let fourth_octet = (index & 0x0f) << 4; + format!("100.{second_octet}.{third_octet}.{fourth_octet}/28") +} + +fn compose_service_names(document: &serde_yaml::Value, path: &Path) -> Result> { + Ok(compose_services(document, path)? + .keys() + .filter_map(serde_yaml::Value::as_str) + .map(str::to_string) + .collect()) +} + +fn compose_service_dependency_names(service: &serde_yaml::Value) -> Result> { + let Some(mapping) = service.as_mapping() else { + return Err(miette::miette!( + "compose service definition is not a mapping" + )); + }; + let Some(depends_on) = mapping.get(yaml_string("depends_on")) else { + return Ok(Vec::new()); + }; + if let Some(sequence) = depends_on.as_sequence() { + return Ok(sequence + .iter() + .filter_map(serde_yaml::Value::as_str) + .map(str::to_string) + .collect()); + } + if let Some(depends_on_map) = depends_on.as_mapping() { + return Ok(depends_on_map + .keys() + .filter_map(serde_yaml::Value::as_str) + .map(str::to_string) + .collect()); + } + Err(miette::miette!( + "compose service has invalid depends_on declaration" + )) +} + +fn retain_compose_service_dependencies( + service: &mut serde_yaml::Value, + keep_services: &BTreeSet, +) -> Result<()> { + let Some(mapping) = service.as_mapping_mut() else { + return Err(miette::miette!( + "compose service definition is not a mapping" + )); + }; + let depends_on_key = yaml_string("depends_on"); + let Some(depends_on) = mapping.get_mut(&depends_on_key) else { + return Ok(()); + }; + match depends_on { + serde_yaml::Value::Sequence(sequence) => { + sequence.retain(|value| { + value + .as_str() + .is_some_and(|name| keep_services.contains(name)) + }); + if sequence.is_empty() { + mapping.remove(&depends_on_key); + } + } + serde_yaml::Value::Mapping(depends_on_map) => { + depends_on_map.retain(|key, _| { + key.as_str() + .is_some_and(|name| keep_services.contains(name)) + }); + if depends_on_map.is_empty() { + mapping.remove(&depends_on_key); + } + } + _ => { + return Err(miette::miette!( + "compose service has invalid depends_on declaration" + )); + } + } + Ok(()) +} + +fn compose_dynamic_root_service_names( + artifact_root: &Path, + assigned_components: &[String], +) -> Result> { + let compose_path = artifact_root.join("compose.yaml"); + let document = read_compose_document(&compose_path)?; + let service_names = compose_service_names(&document, &compose_path)?; + let plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; + let assigned = assigned_components + .iter() + .map(String::as_str) + .collect::>(); + let mut out = BTreeSet::new(); + for target in plan.targets { + if !matches!(target.kind, MeshProvisionTargetKind::Component) + || !assigned.contains(target.config.identity.id.as_str()) + { + continue; + } + let MeshProvisionOutput::Filesystem { dir } = target.output else { + return Err(miette::miette!( + "compose component {} does not use filesystem mesh output", + target.config.identity.id + )); + }; + let sidecar = Path::new(&dir) + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| { + miette::miette!( + "compose component {} has invalid mesh output dir {}", + target.config.identity.id, + dir + ) + })? + .to_string(); + if service_names.contains(&sidecar) { + out.insert(sidecar.clone()); + } + if let Some(program) = sidecar.strip_suffix("-net") + && service_names.contains(program) + { + out.insert(program.to_string()); + } + } + if out.is_empty() { + return Err(miette::miette!( + "compose child artifact {} does not contain services for {:?}", + compose_path.display(), + assigned_components + )); + } + Ok(out) +} + +fn compose_live_service_names( + plan: &SiteControllerRuntimePlan, + published_children: &[SiteControllerRuntimeChildRecord], +) -> Result> { + let site_compose_path = Path::new(&plan.artifact_dir).join("compose.yaml"); + let site_document = read_compose_document(&site_compose_path)?; + let mut live_services = compose_service_names(&site_document, &site_compose_path)?; + for child in published_children { + let child_compose_path = Path::new(&child.artifact_root).join("compose.yaml"); + let child_document = read_compose_document(&child_compose_path)?; + live_services.extend(compose_service_names(&child_document, &child_compose_path)?); + } + Ok(live_services) +} + +fn compose_service_closure( + document: &serde_yaml::Value, + compose_path: &Path, + roots: &BTreeSet, +) -> Result> { + let services = compose_services(document, compose_path)?; + let mut closure = BTreeSet::new(); + let mut queue = roots.iter().cloned().collect::>(); + while let Some(service_name) = queue.pop() { + if !closure.insert(service_name.clone()) { + continue; + } + let service = services.get(yaml_string(&service_name)).ok_or_else(|| { + miette::miette!( + "compose file {} is missing service {}", + compose_path.display(), + service_name + ) + })?; + for dependency in compose_service_dependency_names(service)? { + queue.push(dependency); + } + } + Ok(closure) +} + +fn replace_artifact_snapshot(root: &Path, files: &BTreeMap) -> Result<()> { + fs::create_dir_all(root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", root.display()))?; + let expected = files.keys().map(PathBuf::from).collect::>(); + for path in walk_files(root)? { + let relative = path + .strip_prefix(root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to relativize {}", path.display()))? + .to_path_buf(); + if !expected.contains(&relative) { + fs::remove_file(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to remove stale artifact {}", path.display()))?; + } + } + for (relative, contents) in files { + let path = root.join(relative); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + fs::write(&path, contents) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display()))?; + } + Ok(()) +} + +fn wait_for_named_kubernetes_resource_deleted( + context: Option<&str>, + namespace: &str, + resource_kind: &str, + name: &str, + label: &str, +) -> Result<()> { + let resource = format!("{resource_kind}/{name}"); + let timeout = format!("{}s", KUBERNETES_WORKLOAD_READY_TIMEOUT.as_secs().max(1)); + let output = kubectl_command(context) + .args([ + "-n", + namespace, + "wait", + "--for=delete", + "--timeout", + timeout.as_str(), + resource.as_str(), + ]) + .output() + .into_diagnostic() + .wrap_err_with(|| format!("{label}: wait for {resource} deletion"))?; + if output.status.success() { + return Ok(()); + } + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if stderr.contains("not found") { + return Ok(()); + } + let detail = if stderr.is_empty() { + format!("status {}", output.status) + } else { + stderr + }; + Err(miette::miette!("{label} failed: {detail}")) +} + +fn wait_for_kubernetes_artifact_workloads_deleted( + context: Option<&str>, + namespace: &str, + workloads: &KubernetesArtifactWorkloads, + site_id: &str, +) -> Result<()> { + let label = format!("wait for kubernetes child workload deletion on site `{site_id}`"); + for job in &workloads.jobs { + wait_for_named_kubernetes_resource_deleted(context, namespace, "job", job, &label)?; + } + for deployment in &workloads.deployments { + wait_for_named_kubernetes_resource_deleted( + context, + namespace, + "deployment", + deployment, + &label, + )?; + } + Ok(()) +} + +async fn wait_for_kubernetes_site_router_ready( + plan: &SiteControllerRuntimePlan, + timeout: Duration, +) -> Result<()> { + debug_assert_eq!(plan.kind, SiteKind::Kubernetes); + + let state_path = Path::new(&plan.site_state_root).join("manager-state.json"); + let deadline = Instant::now() + timeout; + loop { + if state_path.is_file() { + let state: SiteManagerState = read_json(&state_path, "site manager state")?; + if matches!(state.status, SiteLifecycleStatus::Failed) { + return Err(miette::miette!( + "kubernetes site `{}` failed while waiting for router recovery: {}", + plan.site_id, + state + .last_error + .unwrap_or_else(|| "unknown failure".to_string()) + )); + } + if matches!(state.status, SiteLifecycleStatus::Running) + && let (Some(control), Some(mesh_addr)) = ( + state.router_control.as_deref(), + state.router_mesh_addr.as_deref(), + ) + { + let control_addr: SocketAddr = + control.parse().into_diagnostic().wrap_err_with(|| { + format!("invalid kubernetes router control addr `{control}`") + })?; + let mesh_addr: SocketAddr = + mesh_addr.parse().into_diagnostic().wrap_err_with(|| { + format!("invalid kubernetes router mesh addr `{mesh_addr}`") + })?; + if probe_kubernetes_router_identity(control_addr, Duration::from_millis(250)) + .await? + && router_mesh_listener_ready(mesh_addr).await + { + return Ok(()); + } + } + } else if let Some(control_target) = plan.local_router_control.as_deref() { + let mesh_target = kubernetes_local_router_mesh_target(plan)?; + if probe_kubernetes_router_identity_target(control_target, Duration::from_millis(250)) + .await? + && router_mesh_listener_ready_target(&mesh_target).await + { + return Ok(()); + } + } + if Instant::now() >= deadline { + return Err(miette::miette!( + "timed out waiting for kubernetes site `{}` router recovery", + plan.site_id + )); + } + sleep(Duration::from_millis(200)).await; + } +} + +fn kubernetes_local_router_mesh_target(plan: &SiteControllerRuntimePlan) -> Result { + let control_target = plan.local_router_control.as_deref().ok_or_else(|| { + miette::miette!( + "kubernetes site `{}` is missing its local router control endpoint", + plan.site_id + ) + })?; + let (host, _) = control_target.rsplit_once(':').ok_or_else(|| { + miette::miette!( + "kubernetes site `{}` has invalid local router control endpoint `{control_target}`", + plan.site_id + ) + })?; + let mesh_port = plan.router_mesh_port.ok_or_else(|| { + miette::miette!( + "kubernetes site `{}` is missing its router mesh port", + plan.site_id + ) + })?; + Ok(format!("{host}:{mesh_port}")) +} + +async fn router_mesh_listener_ready_target(target: &str) -> bool { + tokio::net::TcpStream::connect(target).await.is_ok() +} + +async fn probe_kubernetes_router_identity(addr: SocketAddr, timeout: Duration) -> Result { + probe_kubernetes_router_identity_target(&addr.to_string(), timeout).await +} + +async fn probe_kubernetes_router_identity_target(target: &str, timeout: Duration) -> Result { + let mut stream = + match tokio::time::timeout(timeout, tokio::net::TcpStream::connect(target)).await { + Ok(Ok(stream)) => stream, + Ok(Err(err)) + if matches!( + err.kind(), + std::io::ErrorKind::ConnectionRefused + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::TimedOut + | std::io::ErrorKind::UnexpectedEof + | std::io::ErrorKind::NotConnected + ) => + { + return Ok(false); + } + Ok(Err(err)) => { + return Err(miette::miette!( + "failed to connect to kubernetes router control at {target}: {err}" + )); + } + Err(_) => return Ok(false), + }; + + let request = b"GET /identity HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"; + match tokio::time::timeout(timeout, stream.write_all(request)).await { + Ok(Ok(())) => {} + Ok(Err(err)) + if matches!( + err.kind(), + std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::TimedOut + | std::io::ErrorKind::UnexpectedEof + | std::io::ErrorKind::NotConnected + ) => + { + return Ok(false); + } + Ok(Err(err)) => { + return Err(miette::miette!( + "failed to write kubernetes router identity probe to {target}: {err}" + )); + } + Err(_) => return Ok(false), + } + + let deadline = Instant::now() + timeout; + let mut buf = Vec::new(); + loop { + if let Some(end) = find_header_end(&buf) + && let Some(content_length) = router_identity_probe_content_length(&buf[..end])? + { + let body_len = buf.len().saturating_sub(end + 4); + if body_len >= content_length { + return parse_router_identity_probe_response(&buf, end); + } + } + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return Ok(false); + }; + let mut chunk = [0u8; 1024]; + match tokio::time::timeout(remaining, stream.read(&mut chunk)).await { + Ok(Ok(0)) => return Ok(false), + Ok(Ok(read)) => buf.extend_from_slice(&chunk[..read]), + Ok(Err(err)) + if matches!( + err.kind(), + std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::TimedOut + | std::io::ErrorKind::UnexpectedEof + | std::io::ErrorKind::NotConnected + ) => + { + return Ok(false); + } + Ok(Err(err)) => { + return Err(miette::miette!( + "failed to read kubernetes router identity probe from {target}: {err}" + )); + } + Err(_) => return Ok(false), + } + } +} + +fn router_identity_probe_content_length(header: &[u8]) -> Result> { + let header = std::str::from_utf8(header) + .into_diagnostic() + .wrap_err("router identity probe returned a non-UTF-8 HTTP header")?; + Ok(header.lines().find_map(|line| { + let (name, value) = line.split_once(':')?; + name.trim() + .eq_ignore_ascii_case("content-length") + .then_some(value.trim()) + .and_then(|value| value.parse::().ok()) + })) +} + +fn parse_router_identity_probe_response(buf: &[u8], header_end: usize) -> Result { + let header = std::str::from_utf8(&buf[..header_end]) + .into_diagnostic() + .wrap_err("router identity probe returned a non-UTF-8 HTTP header")?; + let status = header + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .and_then(|code| code.parse::().ok()); + if status != Some(200) { + return Ok(false); + } + let body = std::str::from_utf8(&buf[header_end + 4..]) + .into_diagnostic() + .wrap_err("router identity probe returned a non-UTF-8 body")?; + Ok(serde_json::from_str::(body.trim()).is_ok()) +} + +fn filter_direct_stage_plan(artifact_root: &Path, component_ids: &[usize]) -> Result<()> { + let keep = component_ids.iter().copied().collect::>(); + let plan_path = artifact_root.join("direct-plan.json"); + let mut plan: DirectPlan = read_json(&plan_path, "direct plan")?; + plan.components + .retain(|component| keep.contains(&component.id)); + plan.startup_order + .retain(|component_id| keep.contains(component_id)); + filter_dynamic_runtime_addresses(&mut plan.runtime_addresses, &keep); + let keep_mesh_output_dirs = plan + .components + .iter() + .filter_map(|component| { + Path::new(&component.sidecar.mesh_config_path) + .parent() + .map(path_to_forward_slash_string) + }) + .collect::>(); + plan.router = None; + write_json(&plan_path, &plan)?; + filter_dynamic_mesh_provision_plan(artifact_root, &keep_mesh_output_dirs) +} + +fn filter_vm_stage_plan(artifact_root: &Path, component_ids: &[usize]) -> Result<()> { + let keep = component_ids.iter().copied().collect::>(); + let plan_path = artifact_root.join("vm-plan.json"); + let mut plan: VmPlan = read_json(&plan_path, "vm plan")?; + plan.components + .retain(|component| keep.contains(&component.id)); + plan.startup_order + .retain(|component_id| keep.contains(component_id)); + filter_dynamic_runtime_addresses(&mut plan.runtime_addresses, &keep); + let keep_mesh_output_dirs = plan + .components + .iter() + .filter_map(|component| { + Path::new(&component.mesh_config_path) + .parent() + .map(path_to_forward_slash_string) + }) + .collect::>(); + plan.router = None; + write_json(&plan_path, &plan)?; + filter_dynamic_mesh_provision_plan(artifact_root, &keep_mesh_output_dirs) +} + +fn filter_dynamic_runtime_addresses( + runtime_addresses: &mut amber_compiler::reporter::direct::DirectRuntimeAddressPlan, + keep: &BTreeSet, +) { + runtime_addresses + .slots_by_scope + .retain(|scope, _| keep.contains(scope)); + for sources in runtime_addresses.slots_by_scope.values_mut() { + sources.retain(|_, source| keep.contains(&dynamic_runtime_source_component_id(source))); + } + runtime_addresses + .slot_items_by_scope + .retain(|scope, _| keep.contains(scope)); + for sources in runtime_addresses.slot_items_by_scope.values_mut() { + sources.retain(|_, items| { + items.retain(|source| keep.contains(&dynamic_runtime_source_component_id(source))); + !items.is_empty() + }); + } +} + +fn dynamic_runtime_source_component_id(source: &DirectRuntimeUrlSource) -> usize { + match source { + DirectRuntimeUrlSource::Slot { component_id, .. } + | DirectRuntimeUrlSource::SlotItem { component_id, .. } => *component_id, + } +} + +fn path_to_forward_slash_string(path: &Path) -> String { + path.components() + .fold(String::new(), |mut rendered, component| { + if !rendered.is_empty() { + rendered.push('/'); + } + rendered.push_str(&component.as_os_str().to_string_lossy()); + rendered + }) +} + +fn filter_dynamic_mesh_provision_plan( + artifact_root: &Path, + keep_component_output_dirs: &BTreeSet, +) -> Result<()> { + let plan_path = artifact_root.join("mesh-provision-plan.json"); + let mut plan: MeshProvisionPlan = read_json(&plan_path, "mesh provision plan")?; + let mut kept_component_ids = BTreeSet::new(); + let mut router_target = None; + let mut filtered_targets = Vec::with_capacity(plan.targets.len()); + for target in plan.targets { + match target.kind { + MeshProvisionTargetKind::Component => { + let MeshProvisionOutput::Filesystem { dir } = &target.output else { + return Err(miette::miette!( + "dynamic mesh target {} does not use filesystem output", + target.config.identity.id + )); + }; + if keep_component_output_dirs.contains(dir) { + kept_component_ids.insert(target.config.identity.id.clone()); + filtered_targets.push(target); + } + } + MeshProvisionTargetKind::Router => { + router_target = Some(target); + } + } + } + + let Some(mut router_target) = router_target else { + return Err(miette::miette!( + "dynamic mesh provision plan {} is missing a router target", + plan_path.display() + )); + }; + filter_dynamic_router_target(&mut router_target, &kept_component_ids); + filtered_targets.push(router_target); + plan.targets = filtered_targets; + write_json(&plan_path, &plan) +} + +async fn wait_for_detached_child_runtime_state( + pid: u32, + state_path: &Path, + timeout: Duration, + log_path: &Path, +) -> Result<()> { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if state_path.is_file() { + return Ok(()); + } + if !pid_is_alive(pid) { + let log = fs::read_to_string(log_path).unwrap_or_default(); + return Err(miette::miette!( + "dynamic child runtime exited before becoming ready\nlog ({}):\n{}", + log_path.display(), + log + )); + } + sleep(Duration::from_millis(100)).await; + } + let log = fs::read_to_string(log_path).unwrap_or_default(); + Err(miette::miette!( + "timed out waiting for dynamic child runtime state {}\nlog ({}):\n{}", + state_path.display(), + log_path.display(), + log + )) +} + +fn wait_for_detached_vm_child_endpoints_ready( + pid: u32, + artifact_root: &Path, + runtime_root: &Path, + timeout: Duration, + log_path: &Path, +) -> Result<()> { + let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; + let deadline = Instant::now() + timeout; + for component in &plan.components { + let config: MeshConfigPublic = read_json( + &runtime_root.join(&component.mesh_config_path), + "mesh config", + )?; + for route in config.inbound { + let InboundTarget::Local { port: host_port } = route.target else { + continue; + }; + let addr = SocketAddr::from(([127, 0, 0, 1], host_port)); + loop { + let ready = match route.protocol { + MeshProtocol::Http => { + endpoint_returns_http_response_blocking(addr, Duration::from_millis(250))? + } + MeshProtocol::Tcp => endpoint_accepts_stable_connection_blocking( + addr, + Duration::from_millis(250), + )?, + }; + if ready { + break; + } + if !pid_is_alive(pid) { + let log = fs::read_to_string(log_path).unwrap_or_default(); + return Err(miette::miette!( + "dynamic vm child runtime exited before component {} endpoint {} became \ + ready\nlog ({}):\n{}", + component.moniker, + addr, + log_path.display(), + log + )); + } + if Instant::now() >= deadline { + let log = fs::read_to_string(log_path).unwrap_or_default(); + let protocol = match route.protocol { + MeshProtocol::Http => "http", + MeshProtocol::Tcp => "tcp", + }; + return Err(miette::miette!( + "timed out waiting for dynamic vm child component {} {} endpoint {} to \ + become ready\nlog ({}):\n{}", + component.moniker, + protocol, + addr, + log_path.display(), + log + )); + } + std::thread::sleep(Duration::from_millis(50)); + } + } + } + Ok(()) +} + +fn local_direct_peer_ports( + plan: &SiteControllerRuntimePlan, + state: &SiteControllerRuntimeState, +) -> Result> { + let children = state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>(); + local_direct_peer_ports_for_children(plan, &children) +} + +fn local_direct_peer_ports_for_children( + plan: &SiteControllerRuntimePlan, + children: &[SiteControllerRuntimeChildRecord], +) -> Result> { + let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { + miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) + })?); + let mut peers = BTreeMap::new(); + peers.insert( + plan.router_identity_id.clone(), + plan.router_mesh_port.ok_or_else(|| { + miette::miette!( + "direct site `{}` is missing its router mesh port", + plan.site_id + ) + })?, + ); + peers.extend(direct_peer_ports_for_artifact( + Path::new(&plan.artifact_dir), + site_runtime_root, + )?); + for child in children { + peers.extend(direct_peer_ports_for_artifact( + Path::new(&child.artifact_root), + &site_controller_runtime_child_runtime_root(plan, child.child_id), + )?); + } + Ok(peers) +} + +fn local_direct_peer_identities( + plan: &SiteControllerRuntimePlan, + state: &SiteControllerRuntimeState, +) -> Result> { + let children = state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>(); + local_direct_peer_identities_for_children(plan, &children) +} + +fn local_direct_peer_identities_for_children( + plan: &SiteControllerRuntimePlan, + children: &[SiteControllerRuntimeChildRecord], +) -> Result> { + let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { + miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) + })?); + let mut peers = BTreeMap::new(); + let router = + direct_router_identity_for_artifact(Path::new(&plan.artifact_dir), site_runtime_root)? + .ok_or_else(|| { + miette::miette!( + "direct site `{}` is missing its live router identity", + plan.site_id + ) + })?; + peers.insert(router.id.clone(), router); + peers.extend(direct_peer_identities_for_artifact( + Path::new(&plan.artifact_dir), + site_runtime_root, + )?); + for child in children { + peers.extend(direct_peer_identities_for_artifact( + Path::new(&child.artifact_root), + &site_controller_runtime_child_runtime_root(plan, child.child_id), + )?); + } + Ok(peers) +} + +fn direct_peer_ports_for_artifact( + artifact_root: &Path, + runtime_root: &Path, +) -> Result> { + let plan: DirectPlan = read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; + let mut peers = BTreeMap::new(); + for component in &plan.components { + let config: MeshConfigPublic = read_json( + &runtime_root.join(&component.sidecar.mesh_config_path), + "mesh config", + )?; + peers.insert(config.identity.id, config.mesh_listen.port()); + } + Ok(peers) +} + +fn direct_peer_identities_for_artifact( + artifact_root: &Path, + runtime_root: &Path, +) -> Result> { + let plan: DirectPlan = read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; + let mut peers = BTreeMap::new(); + for component in &plan.components { + let config: MeshConfigPublic = read_json( + &runtime_root.join(&component.sidecar.mesh_config_path), + "mesh config", + )?; + peers.insert(config.identity.id.clone(), config.identity); + } + Ok(peers) +} + +fn direct_router_identity_for_artifact( + artifact_root: &Path, + runtime_root: &Path, +) -> Result> { + let plan: DirectPlan = read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; + let Some(router) = plan.router.as_ref() else { + return Ok(None); + }; + let config: MeshConfigPublic = + read_json(&runtime_root.join(&router.mesh_config_path), "mesh config")?; + Ok(Some(config.identity)) +} + +fn local_vm_peer_ports( + plan: &SiteControllerRuntimePlan, + state: &SiteControllerRuntimeState, +) -> Result> { + let children = state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>(); + local_vm_peer_ports_for_children(plan, &children) +} + +fn local_vm_peer_ports_for_children( + plan: &SiteControllerRuntimePlan, + children: &[SiteControllerRuntimeChildRecord], +) -> Result> { + let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { + miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) + })?); + let mut peers = BTreeMap::new(); + peers.insert( + plan.router_identity_id.clone(), + plan.router_mesh_port.ok_or_else(|| { + miette::miette!("vm site `{}` is missing its router mesh port", plan.site_id) + })?, + ); + peers.extend(vm_peer_ports_for_artifact( + Path::new(&plan.artifact_dir), + site_runtime_root, + )?); + for child in children { + peers.extend(vm_peer_ports_for_artifact( + Path::new(&child.artifact_root), + &site_controller_runtime_child_runtime_root(plan, child.child_id), + )?); + } + Ok(peers) +} + +fn local_vm_peer_identities( + plan: &SiteControllerRuntimePlan, + state: &SiteControllerRuntimeState, +) -> Result> { + let children = state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>(); + local_vm_peer_identities_for_children(plan, &children) +} + +fn local_vm_peer_identities_for_children( + plan: &SiteControllerRuntimePlan, + children: &[SiteControllerRuntimeChildRecord], +) -> Result> { + let site_runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { + miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) + })?); + let mut peers = BTreeMap::new(); + let router = vm_router_identity_for_artifact(Path::new(&plan.artifact_dir), site_runtime_root)? + .ok_or_else(|| { + miette::miette!( + "vm site `{}` is missing its live router identity", + plan.site_id + ) + })?; + peers.insert(router.id.clone(), router); + peers.extend(vm_peer_identities_for_artifact( + Path::new(&plan.artifact_dir), + site_runtime_root, + )?); + for child in children { + peers.extend(vm_peer_identities_for_artifact( + Path::new(&child.artifact_root), + &site_controller_runtime_child_runtime_root(plan, child.child_id), + )?); + } + Ok(peers) +} + +fn vm_peer_ports_for_artifact( + artifact_root: &Path, + runtime_root: &Path, +) -> Result> { + let state = load_vm_runtime_state_for_artifact(artifact_root, runtime_root)?; + let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; + let mut peers = BTreeMap::new(); + for component in &plan.components { + let config: MeshConfigPublic = read_json( + &runtime_root.join(&component.mesh_config_path), + "mesh config", + )?; + let port = state + .component_mesh_port_by_id + .get(&component.id) + .copied() + .ok_or_else(|| { + miette::miette!( + "vm runtime state is missing mesh port for component {}", + component.moniker + ) + })?; + peers.insert(config.identity.id, port); + } + Ok(peers) +} + +fn load_vm_runtime_state_for_artifact( + artifact_root: &Path, + runtime_root: &Path, +) -> Result { + let state_path = artifact_root.join(".amber").join("vm-runtime.json"); + if state_path.is_file() { + return read_json(&state_path, "vm runtime state"); + } + + let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; + let mut state = VmRuntimeState::default(); + for component in &plan.components { + let config: MeshConfigPublic = read_json( + &runtime_root.join(&component.mesh_config_path), + "mesh config", + )?; + state + .component_mesh_port_by_id + .insert(component.id, config.mesh_listen.port()); + } + if let Some(router) = &plan.router { + let config: MeshConfigPublic = + read_json(&runtime_root.join(&router.mesh_config_path), "mesh config")?; + state.router_mesh_port = Some(config.mesh_listen.port()); + } + write_vm_runtime_state(artifact_root, &state)?; + Ok(state) +} + +fn vm_peer_identities_for_artifact( + artifact_root: &Path, + runtime_root: &Path, +) -> Result> { + let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; + let mut peers = BTreeMap::new(); + for component in &plan.components { + let config: MeshConfigPublic = read_json( + &runtime_root.join(&component.mesh_config_path), + "mesh config", + )?; + peers.insert(config.identity.id.clone(), config.identity); + } + Ok(peers) +} + +fn vm_router_identity_for_artifact( + artifact_root: &Path, + runtime_root: &Path, +) -> Result> { + let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; + let Some(router) = plan.router.as_ref() else { + return Ok(None); + }; + let config: MeshConfigPublic = + read_json(&runtime_root.join(&router.mesh_config_path), "mesh config")?; + Ok(Some(config.identity)) +} + +fn terminate_pid(pid: u32, timeout: Duration) -> Result<()> { + #[cfg(unix)] + { + terminate_detached_runtime(pid, timeout) + } + + #[cfg(not(unix))] + { + send_sigterm(pid); + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if !pid_is_alive(pid) { + return Ok(()); + } + std::thread::sleep(Duration::from_millis(100)); + } + Ok(()) + } +} + +#[cfg(unix)] +fn terminate_detached_runtime(root_pid: u32, timeout: Duration) -> Result<()> { + let mut tracked = process_tree_postorder(root_pid)? + .into_iter() + .filter(|pid| pid_is_alive(*pid)) + .collect::>(); + if tracked.is_empty() { + return Ok(()); + } + + send_signal_to_process_group(root_pid, libc::SIGTERM); + send_signal_to_pids(&tracked, libc::SIGTERM); + + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + tracked.retain(|pid| pid_is_alive(*pid)); + if tracked.is_empty() { + return Ok(()); + } + std::thread::sleep(Duration::from_millis(100)); + } + + send_signal_to_process_group(root_pid, libc::SIGKILL); + send_signal_to_pids(&tracked, libc::SIGKILL); + + let force_deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < force_deadline { + tracked.retain(|pid| pid_is_alive(*pid)); + if tracked.is_empty() { + return Ok(()); + } + std::thread::sleep(Duration::from_millis(100)); + } + + Err(miette::miette!( + "failed to terminate detached runtime rooted at pid {root_pid}; surviving processes: {}", + tracked + .iter() + .map(u32::to_string) + .collect::>() + .join(", ") + )) +} diff --git a/runtime/site-controller/src/default_runtime/child_runtime.rs b/runtime/site-controller/src/default_runtime/child_runtime.rs new file mode 100644 index 00000000..c8966f8e --- /dev/null +++ b/runtime/site-controller/src/default_runtime/child_runtime.rs @@ -0,0 +1,1139 @@ +use super::*; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(super) struct SiteControllerRuntimeState { + pub(super) schema: String, + pub(super) version: u32, + pub(super) run_id: String, + pub(super) site_id: String, + pub(super) kind: SiteKind, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub(super) children: BTreeMap, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(super) struct SiteControllerRuntimeChildRecord { + pub(super) child_id: u64, + pub(super) artifact_root: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(super) assigned_components: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub(super) proxy_exports: BTreeMap, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(super) direct_inputs: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(super) routed_inputs: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(super) process_pid: Option, + pub(super) published: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(super) struct DynamicComposeChildMetadata { + pub(super) schema: String, + pub(super) version: u32, + pub(super) services: Vec, + pub(super) readiness_services: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(super) struct StoredRouteOverlayPayload { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(super) peers: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(super) inbound_routes: Vec, +} + +pub(crate) struct BridgeProxyHandle { + pub(crate) child: Child, + pub(crate) listen: SocketAddr, +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct BridgeProxyKey { + pub(crate) provider_output_dir: String, + pub(crate) export_name: String, + pub(crate) consumer_kind: SiteKind, +} + +#[derive(Clone, Default)] +pub struct DefaultSiteControllerRuntime { + bridge_proxies: Arc>>, +} + +pub(crate) fn default_site_controller_runtime() -> SharedSiteControllerRuntime { + Arc::new(DefaultSiteControllerRuntime::default()) +} + +impl SiteControllerRuntime for DefaultSiteControllerRuntime { + fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async move { + let mut bridge_proxies = { + let mut guard = self.bridge_proxies.lock().await; + std::mem::take(&mut *guard) + }; + stop_bridge_proxies(&mut bridge_proxies).await + }) + } + + fn resolve_link_external_url<'a>( + &'a self, + provider: &'a LaunchedSite, + provider_output_dir: &'a Path, + link: &'a RunLink, + consumer_kind: SiteKind, + run_root: &'a Path, + ) -> SiteControllerRuntimeFuture<'a, String> { + Box::pin(async move { + let mut bridge_proxies = self.bridge_proxies.lock().await; + resolve_link_external_url_for_output( + provider, + provider_output_dir, + link, + consumer_kind, + run_root, + &mut bridge_proxies, + ) + .await + }) + } + + fn prepare_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + state: FrameworkControlState, + child: LiveChildRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async move { site_controller_prepare_child(plan, state, child).await }) + } + + fn publish_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + state: FrameworkControlState, + child: LiveChildRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async move { site_controller_publish_child(plan, state, child).await }) + } + + fn rollback_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + child_id: u64, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async move { site_controller_rollback_child(plan, child_id).await }) + } + + fn destroy_child<'a>( + &'a self, + plan: &'a SiteControllerPlan, + state: FrameworkControlState, + child: LiveChildRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async move { site_controller_destroy_child(plan, state, child).await }) + } + + fn collect_live_component_runtime_metadata( + &self, + plan: &SiteControllerRuntimePlan, + ) -> Result> { + collect_live_component_runtime_metadata(plan) + } + + fn load_live_site_router_mesh_config( + &self, + plan: &SiteControllerRuntimePlan, + ) -> Result { + load_live_site_router_mesh_config(plan) + } + + fn router_mesh_addr_for_consumer( + &self, + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> Result { + router_mesh_addr_for_consumer(provider_kind, consumer_kind, router_mesh_addr) + } + + fn update_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExternalSlotOverlay, + ) -> Result<()> { + update_desired_overlay_for_consumer(site_state_root, overlay_id, overlay) + } + + fn update_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExportPeerOverlay, + ) -> Result<()> { + update_desired_overlay_for_provider(site_state_root, overlay_id, overlay) + } + + fn clear_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> Result<()> { + clear_desired_overlay_for_consumer(site_state_root, overlay_id) + } + + fn clear_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> Result<()> { + clear_desired_overlay_for_provider(site_state_root, overlay_id) + } +} + +pub(crate) async fn stop_bridge_proxies( + bridge_proxies: &mut BTreeMap, +) -> Result<()> { + for bridge in bridge_proxies.values_mut() { + stop_child(&mut bridge.child).await?; + } + bridge_proxies.clear(); + Ok(()) +} + +pub(super) fn build_site_controller_runtime_app( + plan: SiteControllerRuntimePlan, +) -> Result { + let state_path = site_controller_runtime_state_path(Path::new(&plan.site_state_root)); + let initial_state = if state_path.is_file() { + read_json(&state_path, "site controller runtime state")? + } else { + let state = SiteControllerRuntimeState { + schema: SITE_CONTROLLER_RUNTIME_STATE_SCHEMA.to_string(), + version: SITE_CONTROLLER_RUNTIME_STATE_VERSION, + run_id: plan.run_id.clone(), + site_id: plan.site_id.clone(), + kind: plan.kind, + children: BTreeMap::new(), + }; + write_json(&state_path, &state)?; + state + }; + Ok(SiteControllerRuntimeApp { + plan, + state_path, + state: Arc::new(AsyncMutex::new(initial_state)), + }) +} + +pub(crate) async fn site_controller_prepare_child( + plan: &SiteControllerPlan, + state: FrameworkControlState, + child: LiveChildRecord, +) -> Result<()> { + let app = build_site_controller_runtime_app( + crate::runtime_api::site_controller_runtime_plan_from_controller_plan(plan), + )?; + site_controller_runtime_prepare_child(&app, &state, &child).await +} + +pub(crate) async fn site_controller_publish_child( + plan: &SiteControllerPlan, + state: FrameworkControlState, + child: LiveChildRecord, +) -> Result<()> { + let app = build_site_controller_runtime_app( + crate::runtime_api::site_controller_runtime_plan_from_controller_plan(plan), + )?; + site_controller_runtime_publish_child(&app, &state, &child).await +} + +pub(crate) async fn site_controller_rollback_child( + plan: &SiteControllerPlan, + child_id: u64, +) -> Result<()> { + let app = build_site_controller_runtime_app( + crate::runtime_api::site_controller_runtime_plan_from_controller_plan(plan), + )?; + site_controller_runtime_rollback_child(&app, child_id).await +} + +pub(crate) async fn site_controller_destroy_child( + plan: &SiteControllerPlan, + state: FrameworkControlState, + child: LiveChildRecord, +) -> Result<()> { + let app = build_site_controller_runtime_app( + crate::runtime_api::site_controller_runtime_plan_from_controller_plan(plan), + )?; + site_controller_runtime_destroy_child(&app, &state, &child).await +} + +#[derive(Clone)] +pub(super) struct SiteControllerRuntimeApp { + pub(super) plan: SiteControllerRuntimePlan, + pub(super) state_path: PathBuf, + pub(super) state: Arc>, +} + +fn local_child_runtime_spec( + plan: &SiteControllerRuntimePlan, + state: &FrameworkControlState, + child: &LiveChildRecord, +) -> Result { + let site_id = child_runtime_site_id(child).map_err(|err| miette::miette!(err.message))?; + if site_id != plan.site_id { + return Err(miette::miette!( + "dynamic child `{}` targeted site `{site_id}` but runtime plan belongs to site `{}`", + child.name, + plan.site_id + )); + } + build_local_child_runtime_spec(state, child, &site_id) + .map_err(|err| miette::miette!(err.message)) +} + +pub(super) fn site_controller_runtime_state_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("site-controller-runtime-state.json") +} + +pub(super) fn site_controller_runtime_child_root( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> PathBuf { + Path::new(&plan.site_state_root) + .join("framework-component") + .join("children") + .join(child_id.to_string()) +} + +pub(super) fn site_controller_runtime_child_artifact_root( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> PathBuf { + site_controller_runtime_child_root(plan, child_id).join("artifact") +} + +pub(super) fn site_controller_runtime_child_runtime_root( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> PathBuf { + site_controller_runtime_child_root(plan, child_id).join("runtime") +} + +pub(super) fn site_controller_runtime_child_storage_root( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> PathBuf { + site_controller_runtime_child_root(plan, child_id).join("storage") +} + +pub(super) fn site_controller_runtime_child_peer_ports_path( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> PathBuf { + site_controller_runtime_child_root(plan, child_id).join("existing-peer-ports.json") +} + +pub(super) fn site_controller_runtime_child_peer_identities_path( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> PathBuf { + site_controller_runtime_child_root(plan, child_id).join("existing-peer-identities.json") +} + +pub(super) fn dynamic_compose_child_metadata_path(artifact_root: &Path) -> PathBuf { + artifact_root.join(".amber").join("compose-child.json") +} + +pub(super) fn dynamic_route_overlay_path(artifact_root: &Path) -> PathBuf { + artifact_root + .join(".amber") + .join(DYNAMIC_ROUTE_OVERLAY_FILENAME) +} + +pub(super) fn write_dynamic_route_overlay_payload( + artifact_root: &Path, + payload: &StoredRouteOverlayPayload, +) -> Result<()> { + let path = dynamic_route_overlay_path(artifact_root); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + write_json(&path, payload) +} + +fn dynamic_proxy_exports_path(artifact_root: &Path) -> PathBuf { + artifact_root + .join(".amber") + .join(DYNAMIC_PROXY_EXPORTS_FILENAME) +} + +pub(super) fn write_dynamic_proxy_exports_metadata( + artifact_root: &Path, + proxy_exports: &BTreeMap, +) -> Result<()> { + if proxy_exports.is_empty() { + return Ok(()); + } + let path = dynamic_proxy_exports_path(artifact_root); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + write_json(&path, proxy_exports) +} + +pub(super) fn load_dynamic_proxy_exports_metadata( + artifact_root: &Path, +) -> Result> { + let path = dynamic_proxy_exports_path(artifact_root); + if !path.is_file() { + return Ok(BTreeMap::new()); + } + read_json(&path, "dynamic proxy exports") +} + +pub fn cleanup_dynamic_site_children(site_state_root: &Path, kind: SiteKind) -> Result<()> { + let state_path = site_controller_runtime_state_path(site_state_root); + if !state_path.is_file() { + return Ok(()); + } + let mut state: SiteControllerRuntimeState = + read_json(&state_path, "site controller runtime state")?; + let compose_supervisor_plan = matches!(kind, SiteKind::Compose) + .then(|| { + read_json::( + &site_supervisor_plan_path(site_state_root), + "site supervisor plan", + ) + }) + .transpose()?; + for child in state.children.values() { + if let Some(plan) = compose_supervisor_plan.as_ref() { + cleanup_dynamic_compose_child(plan, child)?; + } + if let Some(pid) = child.process_pid { + terminate_pid(pid, site_ready_timeout_for_kind(kind))?; + } + remove_dir_if_exists( + &crate::runtime_api::site_controller_runtime_child_root_for_site( + site_state_root, + child.child_id, + ), + )?; + } + if state.children.is_empty() { + return Ok(()); + } + state.children.clear(); + write_json(&state_path, &state) +} + +fn cleanup_dynamic_compose_child( + plan: &SiteSupervisorPlan, + child: &SiteControllerRuntimeChildRecord, +) -> Result<()> { + let artifact_root = Path::new(&child.artifact_root); + if !dynamic_compose_child_metadata_path(artifact_root).is_file() { + return Ok(()); + } + let compose_project = plan.compose_project.as_deref().ok_or_else(|| { + miette::miette!( + "compose site `{}` is missing its compose project name", + plan.site_id + ) + })?; + let status = compose_command(Some(compose_project), artifact_root) + .envs(plan.launch_env.clone()) + .arg("down") + .arg("-v") + .status() + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to clean up dynamic compose child for site `{}`", + plan.site_id + ) + })?; + if !status.success() { + return Err(miette::miette!( + "dynamic compose child cleanup on site `{}` failed with status {status}", + plan.site_id + )); + } + Ok(()) +} + +pub(super) async fn site_controller_runtime_prepare_child( + app: &SiteControllerRuntimeApp, + state: &FrameworkControlState, + child: &LiveChildRecord, +) -> Result<()> { + let runtime_spec = local_child_runtime_spec(&app.plan, state, child)?; + let artifact_root = site_controller_runtime_child_artifact_root(&app.plan, child.child_id); + let published_children = { + let state = app.state.lock().await; + state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>() + }; + let live_components = if runtime_spec.direct_inputs.is_empty() { + BTreeMap::new() + } else { + collect_live_component_runtime_metadata(&app.plan)? + }; + replace_artifact_snapshot(&artifact_root, &runtime_spec.child_artifact_files)?; + project_dynamic_child_mesh_scope(&artifact_root, Some(&app.plan.mesh_scope))?; + if app.plan.kind == SiteKind::Kubernetes { + let _ = prepare_kubernetes_artifact_namespace( + &app.plan.run_id, + &app.plan.site_id, + &artifact_root, + )?; + } + patch_site_artifacts( + &artifact_root, + &app.plan.run_id, + &app.plan.site_id, + app.plan.kind, + &app.plan.launch_env, + app.plan.observability_endpoint.as_deref(), + )?; + match app.plan.kind { + SiteKind::Direct => { + filter_direct_stage_plan(&artifact_root, &runtime_spec.component_ids)?; + ensure_dynamic_proxy_export_component_routes_in_artifact( + &artifact_root, + &runtime_spec.proxy_exports, + &app.plan.router_identity_id, + )?; + rewrite_dynamic_direct_inputs_in_artifact( + &artifact_root, + &runtime_spec.direct_inputs, + &live_components, + )?; + rewrite_dynamic_routed_inputs_in_artifact( + &artifact_root, + &runtime_spec.routed_inputs, + app.plan.kind, + &app.plan.router_identity_id, + app.plan.router_mesh_port, + )?; + write_direct_vm_startup_route_overlay_payload( + &artifact_root, + "direct", + &runtime_spec.routed_inputs, + &overlay_peer_addr_map_from_ports(&local_direct_peer_ports_for_children( + &app.plan, + &published_children, + )?), + &local_direct_peer_identities_for_children(&app.plan, &published_children)?, + )?; + } + SiteKind::Vm => { + filter_vm_stage_plan(&artifact_root, &runtime_spec.component_ids)?; + ensure_dynamic_proxy_export_component_routes_in_artifact( + &artifact_root, + &runtime_spec.proxy_exports, + &app.plan.router_identity_id, + )?; + rewrite_dynamic_direct_inputs_in_artifact( + &artifact_root, + &runtime_spec.direct_inputs, + &live_components, + )?; + rewrite_dynamic_routed_inputs_in_artifact( + &artifact_root, + &runtime_spec.routed_inputs, + app.plan.kind, + &app.plan.router_identity_id, + app.plan.router_mesh_port, + )?; + write_direct_vm_startup_route_overlay_payload( + &artifact_root, + "vm", + &runtime_spec.routed_inputs, + &overlay_peer_addr_map_from_ports(&local_vm_peer_ports_for_children( + &app.plan, + &published_children, + )?), + &local_vm_peer_identities_for_children(&app.plan, &published_children)?, + )?; + } + SiteKind::Compose => { + let existing_peer_identities = + local_compose_peer_identities(&app.plan, &published_children)?; + prepare_dynamic_compose_child_artifact( + &app.plan, + &runtime_spec, + &artifact_root, + &published_children, + &existing_peer_identities, + &live_components, + )? + } + SiteKind::Kubernetes => { + let existing_peer_identities = + local_kubernetes_peer_identities(&app.plan, &published_children)?; + prepare_dynamic_kubernetes_child_artifact( + &app.plan, + &runtime_spec, + &artifact_root, + &existing_peer_identities, + &live_components, + )? + } + } + write_dynamic_proxy_exports_metadata(&artifact_root, &runtime_spec.proxy_exports)?; + rewrite_dynamic_proxy_metadata(&artifact_root, &runtime_spec.proxy_exports, app.plan.kind)?; + if app.plan.kind == SiteKind::Compose + && let Some(compose_project) = app.plan.compose_project.as_deref() + { + rewrite_dynamic_compose_proxy_metadata(&artifact_root, compose_project)?; + } + let mut state = app.state.lock().await; + state.children.insert( + child.child_id, + SiteControllerRuntimeChildRecord { + child_id: child.child_id, + artifact_root: artifact_root.display().to_string(), + assigned_components: runtime_spec.assigned_components.clone(), + proxy_exports: runtime_spec.proxy_exports.clone(), + direct_inputs: runtime_spec.direct_inputs.clone(), + routed_inputs: runtime_spec.routed_inputs.clone(), + process_pid: None, + published: false, + }, + ); + write_json(&app.state_path, &*state) +} + +pub(super) async fn site_controller_runtime_publish_child( + app: &SiteControllerRuntimeApp, + state: &FrameworkControlState, + child: &LiveChildRecord, +) -> Result<()> { + let runtime_spec = local_child_runtime_spec(&app.plan, state, child)?; + let child_id = child.child_id; + let (child, published_children) = { + let state = app.state.lock().await; + let child = state.children.get(&child_id).cloned().ok_or_else(|| { + miette::miette!("site controller runtime child {child_id} is not prepared") + })?; + let published_children = state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>(); + (child, published_children) + }; + if child.published { + return Ok(()); + } + + match app.plan.kind { + SiteKind::Direct => { + let state = app.state.lock().await; + let existing_peer_ports = local_direct_peer_ports(&app.plan, &state)?; + let existing_peer_identities = local_direct_peer_identities(&app.plan, &state)?; + drop(state); + write_json( + &site_controller_runtime_child_peer_ports_path(&app.plan, child_id), + &existing_peer_ports, + )?; + write_json( + &site_controller_runtime_child_peer_identities_path(&app.plan, child_id), + &existing_peer_identities, + )?; + project_dynamic_child_mesh_scope( + Path::new(&child.artifact_root), + existing_peer_identities + .get(&app.plan.router_identity_id) + .and_then(|identity| identity.mesh_scope.as_deref()), + )?; + if dynamic_route_overlay_path(Path::new(&child.artifact_root)).is_file() { + apply_dynamic_site_router_overlay(&app.plan, &child).await?; + } + let runtime_root = site_controller_runtime_child_runtime_root(&app.plan, child_id); + let storage_root = site_controller_runtime_child_storage_root(&app.plan, child_id); + let child_root = site_controller_runtime_child_root(&app.plan, child_id); + fs::create_dir_all(&runtime_root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", runtime_root.display()))?; + fs::create_dir_all(&storage_root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", storage_root.display()))?; + let process = spawn_detached_child(&child_root, &child_root.join("site.log"), |cmd| { + cmd.arg("run-direct-init") + .arg("--plan") + .arg(Path::new(&child.artifact_root).join("direct-plan.json")) + .arg("--storage-root") + .arg(&storage_root) + .arg("--runtime-root") + .arg(&runtime_root) + .arg("--existing-peer-ports") + .arg(site_controller_runtime_child_peer_ports_path( + &app.plan, child_id, + )) + .arg("--existing-peer-identities") + .arg(site_controller_runtime_child_peer_identities_path( + &app.plan, child_id, + )) + .arg("--skip-router"); + })?; + { + let mut state = app.state.lock().await; + let record = state + .children + .get_mut(&child_id) + .expect("prepared child should remain present"); + record.process_pid = Some(process.id()); + write_json(&app.state_path, &*state)?; + } + wait_for_detached_child_runtime_state( + process.id(), + &direct_runtime_state_path(Path::new(&child.artifact_root)), + site_ready_timeout_for_kind(SiteKind::Direct), + &child_root.join("site.log"), + ) + .await?; + let mut live_children = published_children.clone(); + live_children.push(child.clone()); + let live_peer_ports = overlay_peer_addr_map_from_ports( + &local_direct_peer_ports_for_children(&app.plan, &live_children)?, + ); + let live_peer_identities = + local_direct_peer_identities_for_children(&app.plan, &live_children)?; + write_direct_vm_live_route_overlay_payload( + Path::new(&child.artifact_root), + &child.assigned_components, + &child.proxy_exports, + &child.routed_inputs, + &live_peer_ports, + &live_peer_identities, + )?; + apply_dynamic_site_router_overlay(&app.plan, &child).await?; + let mut state = app.state.lock().await; + { + let record = state + .children + .get_mut(&child_id) + .expect("prepared child should remain present"); + record.process_pid = Some(process.id()); + record.published = true; + } + write_json(&app.state_path, &*state)?; + drop(state); + project_dynamic_direct_router_surface(&app.plan, &child)?; + reconcile_dynamic_site_router_overlays(app).await?; + reconcile_dynamic_direct_input_overlays(app).await?; + reconcile_site_proxy_metadata( + Path::new(&app.plan.artifact_dir), + &runtime_spec.site_artifact_files, + )?; + } + SiteKind::Vm => { + let state = app.state.lock().await; + let existing_peer_ports = local_vm_peer_ports(&app.plan, &state)?; + let existing_peer_identities = local_vm_peer_identities(&app.plan, &state)?; + drop(state); + write_json( + &site_controller_runtime_child_peer_ports_path(&app.plan, child_id), + &existing_peer_ports, + )?; + write_json( + &site_controller_runtime_child_peer_identities_path(&app.plan, child_id), + &existing_peer_identities, + )?; + project_dynamic_child_mesh_scope( + Path::new(&child.artifact_root), + existing_peer_identities + .get(&app.plan.router_identity_id) + .and_then(|identity| identity.mesh_scope.as_deref()), + )?; + if dynamic_route_overlay_path(Path::new(&child.artifact_root)).is_file() { + apply_dynamic_site_router_overlay(&app.plan, &child).await?; + } + let runtime_root = site_controller_runtime_child_runtime_root(&app.plan, child_id); + let storage_root = site_controller_runtime_child_storage_root(&app.plan, child_id); + let child_root = site_controller_runtime_child_root(&app.plan, child_id); + fs::create_dir_all(&runtime_root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", runtime_root.display()))?; + fs::create_dir_all(&storage_root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", storage_root.display()))?; + let process = spawn_detached_child(&child_root, &child_root.join("site.log"), |cmd| { + cmd.arg("run-vm-init") + .arg("--plan") + .arg(Path::new(&child.artifact_root).join("vm-plan.json")) + .arg("--storage-root") + .arg(&storage_root) + .arg("--runtime-root") + .arg(&runtime_root) + .arg("--existing-peer-ports") + .arg(site_controller_runtime_child_peer_ports_path( + &app.plan, child_id, + )) + .arg("--existing-peer-identities") + .arg(site_controller_runtime_child_peer_identities_path( + &app.plan, child_id, + )) + .arg("--skip-router"); + })?; + { + let mut state = app.state.lock().await; + let record = state + .children + .get_mut(&child_id) + .expect("prepared child should remain present"); + record.process_pid = Some(process.id()); + write_json(&app.state_path, &*state)?; + } + wait_for_detached_child_runtime_state( + process.id(), + &Path::new(&child.artifact_root) + .join(".amber") + .join("vm-runtime.json"), + vm_endpoint_forward_ready_timeout(), + &child_root.join("site.log"), + ) + .await?; + wait_for_detached_vm_child_endpoints_ready( + process.id(), + Path::new(&child.artifact_root), + &runtime_root, + vm_endpoint_forward_ready_timeout(), + &child_root.join("site.log"), + )?; + let mut live_children = published_children.clone(); + live_children.push(child.clone()); + let live_peer_ports = overlay_peer_addr_map_from_ports( + &local_vm_peer_ports_for_children(&app.plan, &live_children)?, + ); + let live_peer_identities = + local_vm_peer_identities_for_children(&app.plan, &live_children)?; + write_direct_vm_live_route_overlay_payload( + Path::new(&child.artifact_root), + &child.assigned_components, + &child.proxy_exports, + &child.routed_inputs, + &live_peer_ports, + &live_peer_identities, + )?; + apply_dynamic_site_router_overlay(&app.plan, &child).await?; + let mut state = app.state.lock().await; + { + let record = state + .children + .get_mut(&child_id) + .expect("prepared child should remain present"); + record.process_pid = Some(process.id()); + record.published = true; + } + write_json(&app.state_path, &*state)?; + drop(state); + project_dynamic_vm_router_surface(&app.plan, &child)?; + reconcile_dynamic_site_router_overlays(app).await?; + reconcile_dynamic_direct_input_overlays(app).await?; + reconcile_site_proxy_metadata( + Path::new(&app.plan.artifact_dir), + &runtime_spec.site_artifact_files, + )?; + } + SiteKind::Compose => { + let mut issuer_children = published_children.clone(); + issuer_children.push(child.clone()); + reconcile_dynamic_site_router_overlays_for_children( + app, + &published_children, + &issuer_children, + ) + .await?; + apply_dynamic_site_router_overlay(&app.plan, &child).await?; + let metadata = load_dynamic_compose_child_metadata(Path::new(&child.artifact_root))?; + let compose_project = app.plan.compose_project.as_deref().ok_or_else(|| { + miette::miette!( + "compose site `{}` is missing its compose project name", + app.plan.site_id + ) + })?; + let (sidecar_services, workload_services): (Vec<_>, Vec<_>) = metadata + .services + .iter() + .cloned() + .partition(|service| is_compose_component_sidecar_service(service)); + if !sidecar_services.is_empty() { + let status = + compose_command(Some(compose_project), Path::new(&child.artifact_root)) + .envs(app.plan.launch_env.clone()) + .arg("up") + .arg("-d") + .args(&sidecar_services) + .status() + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to publish compose child sidecars on site `{}`", + app.plan.site_id + ) + })?; + if !status.success() { + return Err(miette::miette!( + "compose child sidecar publish on site `{}` failed with status {status}", + app.plan.site_id + )); + } + wait_for_compose_services_running( + compose_project, + Path::new(&child.artifact_root), + &sidecar_services, + site_ready_timeout_for_kind(SiteKind::Compose), + ) + .await?; + } + if !workload_services.is_empty() { + let status = + compose_command(Some(compose_project), Path::new(&child.artifact_root)) + .envs(app.plan.launch_env.clone()) + .arg("up") + .arg("-d") + .args(&workload_services) + .status() + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to publish compose child workloads on site `{}`", + app.plan.site_id + ) + })?; + if !status.success() { + return Err(miette::miette!( + "compose child workload publish on site `{}` failed with status {status}", + app.plan.site_id + )); + } + } + wait_for_compose_services_running( + compose_project, + Path::new(&child.artifact_root), + &metadata.readiness_services, + site_ready_timeout_for_kind(SiteKind::Compose), + ) + .await?; + let mut state = app.state.lock().await; + let record = state + .children + .get_mut(&child_id) + .expect("prepared child should remain present"); + record.published = true; + write_json(&app.state_path, &*state)?; + drop(state); + reconcile_dynamic_site_router_overlays(app).await?; + reconcile_dynamic_direct_input_overlays(app).await?; + } + SiteKind::Kubernetes => { + let artifact_root = Path::new(&child.artifact_root); + let supervisor_plan = prepare_kubernetes_artifact_for_apply(&app.plan, artifact_root)?; + ensure_kubernetes_namespace(&supervisor_plan)?; + let status = kubectl_command(app.plan.context.as_deref()) + .current_dir(artifact_root) + .arg("apply") + .arg("-k") + .arg(".") + .status() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to publish kubernetes site `{}`", app.plan.site_id) + })?; + if !status.success() { + return Err(miette::miette!( + "kubernetes site `{}` publish failed with status {status}", + app.plan.site_id + )); + } + ensure_kubernetes_workloads_ready(&supervisor_plan)?; + let mut live_children = published_children.clone(); + live_children.push(child.clone()); + let live_peer_identities = local_kubernetes_peer_identities(&app.plan, &live_children)?; + write_kubernetes_live_route_overlay_payload( + artifact_root, + &child.assigned_components, + &child.proxy_exports, + &child.routed_inputs, + &live_peer_identities, + )?; + apply_dynamic_site_router_overlay(&app.plan, &child).await?; + let mut state = app.state.lock().await; + let record = state + .children + .get_mut(&child_id) + .expect("prepared child should remain present"); + record.published = true; + write_json(&app.state_path, &*state)?; + drop(state); + reconcile_dynamic_site_router_overlays(app).await?; + reconcile_dynamic_direct_input_overlays(app).await?; + wait_for_kubernetes_site_router_ready( + &app.plan, + site_ready_timeout_for_kind(SiteKind::Kubernetes), + ) + .await?; + } + } + + Ok(()) +} + +pub(super) async fn site_controller_runtime_rollback_child( + app: &SiteControllerRuntimeApp, + child_id: u64, +) -> Result<()> { + let child = { + let mut state = app.state.lock().await; + let removed = state.children.remove(&child_id); + write_json(&app.state_path, &*state)?; + removed + }; + let Some(child) = child else { + return Ok(()); + }; + if let Some(pid) = child.process_pid { + terminate_pid(pid, site_ready_timeout_for_kind(app.plan.kind))?; + } + remove_dir_if_exists(&site_controller_runtime_child_root(&app.plan, child_id)) +} + +pub(super) async fn site_controller_runtime_destroy_child( + app: &SiteControllerRuntimeApp, + state: &FrameworkControlState, + child: &LiveChildRecord, +) -> Result<()> { + let site_id = child_runtime_site_id(child).map_err(|err| miette::miette!(err.message))?; + let child_id = child.child_id; + let child = { + let state = app.state.lock().await; + state.children.get(&child_id).cloned() + }; + if let Some(child) = child.as_ref() + && matches!( + app.plan.kind, + SiteKind::Direct | SiteKind::Vm | SiteKind::Compose | SiteKind::Kubernetes + ) + { + revoke_dynamic_site_router_overlay(&app.plan, child).await?; + } + if let Some(pid) = child.as_ref().and_then(|child| child.process_pid) { + terminate_pid(pid, site_ready_timeout_for_kind(app.plan.kind))?; + } + + match app.plan.kind { + SiteKind::Compose => { + let child = child.as_ref().ok_or_else(|| { + miette::miette!("site controller runtime child {child_id} is not prepared") + })?; + let metadata = load_dynamic_compose_child_metadata(Path::new(&child.artifact_root))?; + let compose_project = app.plan.compose_project.as_deref().ok_or_else(|| { + miette::miette!( + "compose site `{}` is missing its compose project name", + app.plan.site_id + ) + })?; + if !metadata.services.is_empty() { + let status = + compose_command(Some(compose_project), Path::new(&child.artifact_root)) + .envs(app.plan.launch_env.clone()) + .arg("rm") + .arg("--stop") + .arg("--force") + .arg("-v") + .args(&metadata.services) + .status() + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to destroy compose child on site `{}`", + app.plan.site_id + ) + })?; + if !status.success() { + return Err(miette::miette!( + "compose child destroy on site `{}` failed with status {status}", + app.plan.site_id + )); + } + } + } + SiteKind::Kubernetes => { + let child = child.as_ref().ok_or_else(|| { + miette::miette!("site controller runtime child {child_id} is not prepared") + })?; + let artifact_root = Path::new(&child.artifact_root); + let files = read_artifact_snapshot(artifact_root)?; + let destroy_bundle = project_kubernetes_dynamic_child_destroy_artifact_files(&files)?; + replace_artifact_snapshot(artifact_root, &destroy_bundle)?; + let supervisor_plan = prepare_kubernetes_artifact_for_apply(&app.plan, artifact_root)?; + let workloads = kubernetes_expected_workloads(artifact_root)?; + let status = kubectl_command(app.plan.context.as_deref()) + .current_dir(artifact_root) + .arg("delete") + .arg("-k") + .arg(".") + .arg("--ignore-not-found=true") + .status() + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to destroy kubernetes child on site `{}`", + app.plan.site_id + ) + })?; + if !status.success() { + return Err(miette::miette!( + "kubernetes child destroy on site `{}` failed with status {status}", + app.plan.site_id + )); + } + wait_for_kubernetes_artifact_workloads_deleted( + app.plan.context.as_deref(), + supervisor_plan + .kubernetes_namespace + .as_deref() + .expect("kubernetes supervisor plan should include a namespace"), + &workloads, + &app.plan.site_id, + )?; + wait_for_kubernetes_site_router_ready( + &app.plan, + site_ready_timeout_for_kind(SiteKind::Kubernetes), + ) + .await?; + } + SiteKind::Direct | SiteKind::Vm => { + let site_artifact_files = build_desired_site_artifact_files(state, &site_id) + .map_err(|err| miette::miette!(err.message))?; + reconcile_site_proxy_metadata(Path::new(&app.plan.artifact_dir), &site_artifact_files)?; + } + } + + let mut state = app.state.lock().await; + state.children.remove(&child_id); + write_json(&app.state_path, &*state)?; + drop(state); + if matches!( + app.plan.kind, + SiteKind::Direct | SiteKind::Vm | SiteKind::Compose | SiteKind::Kubernetes + ) { + reconcile_dynamic_site_router_overlays(app).await?; + reconcile_dynamic_direct_input_overlays(app).await?; + } + remove_dir_if_exists(&site_controller_runtime_child_root(&app.plan, child_id)) +} diff --git a/runtime/site-controller/src/default_runtime/compose_controller.rs b/runtime/site-controller/src/default_runtime/compose_controller.rs new file mode 100644 index 00000000..aa204f7f --- /dev/null +++ b/runtime/site-controller/src/default_runtime/compose_controller.rs @@ -0,0 +1,80 @@ +use std::{fs, path::Path}; + +use serde_json::json; + +use super::*; + +const COMPOSE_MESH_NETWORK_NAME: &str = "amber_mesh"; +const COMPOSE_PROVISIONER_SERVICE_NAME: &str = "amber-provisioner"; +const COMPOSE_ROUTER_SERVICE_NAME: &str = "amber-router"; +const COMPOSE_ROUTER_CONTROL_INIT_SERVICE_NAME: &str = "amber-router-control-init"; +const COMPOSE_ROUTER_CONTROL_SOCKET_DIR: &str = "/amber/control"; +const COMPOSE_ROUTER_CONTROL_VOLUME_NAME: &str = "amber-router-control"; +const DOCKER_SOCK_PATH: &str = "/var/run/docker.sock"; + +pub fn inject_compose_site_controller( + artifact_root: &Path, + plan: &SiteControllerPlan, + plan_path: &Path, + controller_image: &str, +) -> Result<()> { + let compose_path = artifact_root.join("compose.yaml"); + let mut document = read_compose_document(&compose_path)?; + let services = compose_services_mut(&mut document, &compose_path)?; + + let mut depends_on = serde_yaml::Mapping::new(); + for (service_name, condition) in [ + ( + COMPOSE_PROVISIONER_SERVICE_NAME, + "service_completed_successfully", + ), + ( + COMPOSE_ROUTER_CONTROL_INIT_SERVICE_NAME, + "service_completed_successfully", + ), + (COMPOSE_ROUTER_SERVICE_NAME, "service_started"), + ] { + if services.contains_key(yaml_string(service_name)) { + depends_on.insert( + yaml_string(service_name), + serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( + yaml_string("condition"), + yaml_string(condition), + )])), + ); + } + } + + let networks = std::collections::BTreeMap::from([( + COMPOSE_MESH_NETWORK_NAME.to_string(), + serde_json::Value::Object(serde_json::Map::new()), + )]); + let service = json!({ + "image": controller_image, + "user": "0:0", + "command": ["--plan", plan_path.display().to_string()], + "networks": networks, + "volumes": [ + format!("{}:{}", plan.run_root, plan.run_root), + format!( + "{COMPOSE_ROUTER_CONTROL_VOLUME_NAME}:{COMPOSE_ROUTER_CONTROL_SOCKET_DIR}" + ), + format!("{DOCKER_SOCK_PATH}:{DOCKER_SOCK_PATH}") + ], + "depends_on": depends_on, + "restart": "unless-stopped" + }); + services.insert( + yaml_string(SITE_CONTROLLER_SERVICE_NAME), + serde_yaml::to_value(service) + .into_diagnostic() + .wrap_err("failed to serialize compose site controller service")?, + ); + + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", compose_path.display()))?; + fs::write(&compose_path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", compose_path.display())) +} diff --git a/runtime/site-controller/src/default_runtime/dynamic_routes.rs b/runtime/site-controller/src/default_runtime/dynamic_routes.rs new file mode 100644 index 00000000..b767c1de --- /dev/null +++ b/runtime/site-controller/src/default_runtime/dynamic_routes.rs @@ -0,0 +1,1696 @@ +use super::*; + +pub(super) fn dynamic_compose_mesh_dir(service_name: &str) -> String { + format!("{DYNAMIC_COMPOSE_MESH_ROOT}/{service_name}") +} + +pub(super) fn filter_dynamic_router_target( + router_target: &mut amber_mesh::MeshProvisionTarget, + kept_component_ids: &BTreeSet, +) { + router_target + .config + .peers + .retain(|peer| kept_component_ids.contains(&peer.id)); + router_target + .config + .inbound + .retain(|route| match &route.target { + InboundTarget::MeshForward { peer_id, .. } => kept_component_ids.contains(peer_id), + _ => route + .allowed_issuers + .iter() + .any(|issuer| kept_component_ids.contains(issuer)), + }); +} + +pub(super) fn dynamic_proxy_export_mesh_protocol( + export: &DynamicProxyExportRecord, +) -> Result { + let protocol = export + .protocol + .parse::() + .map_err(|err| miette::miette!("invalid dynamic proxy export protocol: {err}"))?; + mesh_protocol(protocol) +} + +pub(super) fn dynamic_proxy_export_http_plugins( + export: &DynamicProxyExportRecord, + protocol: MeshProtocol, +) -> Vec { + matches!( + (export.capability_kind.as_str(), protocol), + ("a2a", MeshProtocol::Http) + ) + .then_some(HttpRoutePlugin::A2a) + .into_iter() + .collect() +} + +pub(super) fn dynamic_proxy_export_route_id( + export_name: &str, + export: &DynamicProxyExportRecord, +) -> Result { + Ok(router_dynamic_export_route_id( + &export.component, + export_name, + dynamic_proxy_export_mesh_protocol(export)?, + )) +} + +pub(super) fn dynamic_input_route_mesh_protocol( + input: &DynamicInputRouteRecord, +) -> Result { + let protocol = input + .protocol + .parse::() + .map_err(|err| miette::miette!("invalid dynamic routed-input protocol: {err}"))?; + mesh_protocol(protocol) +} + +pub(super) fn dynamic_input_direct_mesh_protocol( + input: &DynamicInputDirectRecord, +) -> Result { + let protocol = input + .protocol + .parse::() + .map_err(|err| miette::miette!("invalid dynamic direct-input protocol: {err}"))?; + mesh_protocol(protocol) +} + +pub(super) fn dynamic_input_route_route_id( + input: &DynamicInputRouteRecord, + protocol: MeshProtocol, +) -> String { + match &input.target { + DynamicInputRouteTarget::ComponentProvide { provide } => { + component_route_id(&input.provider_component, provide, protocol) + } + } +} + +pub(super) fn dynamic_input_route_capability(input: &DynamicInputRouteRecord) -> String { + match &input.target { + DynamicInputRouteTarget::ComponentProvide { provide } => provide.clone(), + } +} + +pub(super) fn dynamic_input_direct_route_id( + input: &DynamicInputDirectRecord, + protocol: MeshProtocol, +) -> String { + match &input.target { + DynamicInputRouteTarget::ComponentProvide { provide } => { + component_route_id(&input.provider_component, provide, protocol) + } + } +} + +pub(super) fn dynamic_input_direct_capability(input: &DynamicInputDirectRecord) -> String { + match &input.target { + DynamicInputRouteTarget::ComponentProvide { provide } => provide.clone(), + } +} + +pub(super) fn is_compose_component_sidecar_service(service_name: &str) -> bool { + service_name.ends_with("-net") +} + +pub(super) fn dynamic_input_route_http_plugins( + input: &DynamicInputRouteRecord, + protocol: MeshProtocol, +) -> Vec { + dynamic_proxy_export_http_plugins( + &DynamicProxyExportRecord { + component_id: 0, + component: input.provider_component.clone(), + provide: dynamic_input_route_capability(input), + protocol: input.protocol.clone(), + capability_kind: input.capability_kind.clone(), + capability_profile: input.capability_profile.clone(), + target_port: 0, + }, + protocol, + ) +} + +pub(super) fn dynamic_input_direct_http_plugins( + input: &DynamicInputDirectRecord, + protocol: MeshProtocol, +) -> Vec { + dynamic_proxy_export_http_plugins( + &DynamicProxyExportRecord { + component_id: 0, + component: input.provider_component.clone(), + provide: dynamic_input_direct_capability(input), + protocol: input.protocol.clone(), + capability_kind: input.capability_kind.clone(), + capability_profile: input.capability_profile.clone(), + target_port: 0, + }, + protocol, + ) +} + +pub(super) fn overlay_peer_addr_map_from_ports( + ports: &BTreeMap, +) -> BTreeMap { + ports + .iter() + .map(|(component, port)| (component.clone(), format!("127.0.0.1:{port}"))) + .collect() +} + +pub(super) fn overlay_issuer_sets( + routed_inputs: &[DynamicInputRouteRecord], +) -> Result>> { + dynamic_route_issuer_grants(&[SiteControllerRuntimeChildRecord { + child_id: 0, + artifact_root: String::new(), + assigned_components: Vec::new(), + proxy_exports: BTreeMap::new(), + direct_inputs: Vec::new(), + routed_inputs: routed_inputs.to_vec(), + process_pid: None, + published: false, + }]) +} + +pub(super) fn overlay_upsert_peer( + peers: &mut Vec, + peer_identities: &BTreeMap, + peer_id: &str, +) -> Result<()> { + if peers.iter().any(|peer| peer.id == peer_id) { + return Ok(()); + } + let identity = peer_identities.get(peer_id).ok_or_else(|| { + miette::miette!("dynamic route overlay is missing mesh identity for peer {peer_id}") + })?; + peers.push(MeshPeer { + id: identity.id.clone(), + public_key: identity.public_key, + }); + Ok(()) +} + +pub(super) fn overlay_upsert_route(routes: &mut Vec, route: InboundRoute) { + if let Some(existing) = routes + .iter_mut() + .find(|existing| existing.route_id == route.route_id) + { + *existing = route; + } else { + routes.push(route); + } +} + +pub(super) fn routed_input_overlay_route( + input: &DynamicInputRouteRecord, + provider_peer_addr: &str, + allowed_issuers: Vec, +) -> Result { + let protocol = dynamic_input_route_mesh_protocol(input)?; + let (target_route_id, capability) = match &input.target { + DynamicInputRouteTarget::ComponentProvide { provide } => ( + component_route_id(&input.provider_component, provide, protocol), + provide.clone(), + ), + }; + Ok(InboundRoute { + route_id: dynamic_input_route_route_id(input, protocol), + capability: dynamic_input_route_capability(input), + capability_kind: Some(input.capability_kind.clone()), + capability_profile: input.capability_profile.clone(), + protocol, + http_plugins: dynamic_input_route_http_plugins(input, protocol), + target: InboundTarget::MeshForward { + peer_addr: provider_peer_addr.to_string(), + peer_id: input.provider_component.clone(), + route_id: target_route_id, + capability, + }, + allowed_issuers, + }) +} + +pub(super) fn augment_route_overlay_payload( + payload: &mut StoredRouteOverlayPayload, + proxy_exports: &BTreeMap, + routed_inputs: &[DynamicInputRouteRecord], + provider_peer_addrs: &BTreeMap, + peer_identities: &BTreeMap, + allowed_issuers_by_route: Option<&BTreeMap>>, + skip_missing_providers: bool, +) -> Result<()> { + for export in proxy_exports.values() { + overlay_upsert_peer(&mut payload.peers, peer_identities, &export.component)?; + } + add_dynamic_proxy_export_overlay_routes( + &mut payload.inbound_routes, + proxy_exports, + |export| { + provider_peer_addrs + .get(&export.component) + .cloned() + .ok_or_else(|| { + miette::miette!( + "dynamic route overlay is missing a live peer address for {}", + export.component + ) + }) + }, + )?; + + for input in routed_inputs { + let Some(provider_peer_addr) = provider_peer_addrs.get(&input.provider_component) else { + if skip_missing_providers { + continue; + } + return Err(miette::miette!( + "dynamic route overlay is missing a live peer address for {}", + input.provider_component + )); + }; + overlay_upsert_peer( + &mut payload.peers, + peer_identities, + &input.provider_component, + )?; + let route_id = + dynamic_input_route_route_id(input, dynamic_input_route_mesh_protocol(input)?); + let allowed_issuers = allowed_issuers_by_route + .and_then(|issuers| issuers.get(&route_id)) + .map(|issuers| issuers.iter().cloned().collect()) + .unwrap_or_default(); + overlay_upsert_route( + &mut payload.inbound_routes, + routed_input_overlay_route(input, provider_peer_addr, allowed_issuers)?, + ); + } + + Ok(()) +} + +pub(super) fn routed_input_router_peer_addr( + kind: SiteKind, + router_mesh_port: Option, +) -> Result { + let router_mesh_port = router_mesh_port.ok_or_else(|| { + miette::miette!("site {kind:?} is missing its router mesh port for routed child inputs") + })?; + Ok(match kind { + SiteKind::Direct | SiteKind::Vm => format!("127.0.0.1:{router_mesh_port}"), + SiteKind::Compose => format!("{COMPOSE_ROUTER_SERVICE_NAME}:{router_mesh_port}"), + SiteKind::Kubernetes => { + format!("{KUBERNETES_ROUTER_COMPONENT_NAME}:{router_mesh_port}") + } + }) +} + +pub(super) fn router_mesh_port_from_plan( + mesh_plan: &MeshProvisionPlan, + artifact_kind: &str, +) -> Result { + mesh_plan + .targets + .iter() + .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) + .map(|target| target.config.mesh_listen.port()) + .ok_or_else(|| { + miette::miette!("{artifact_kind} mesh provision plan is missing its router mesh target") + }) +} + +pub(super) fn rewrite_dynamic_routed_inputs( + mesh_plan: &mut MeshProvisionPlan, + routed_inputs: &[DynamicInputRouteRecord], + kind: SiteKind, + router_identity_id: &str, + router_mesh_port: Option, +) -> Result<()> { + if routed_inputs.is_empty() { + return Ok(()); + } + + let router_peer_addr = routed_input_router_peer_addr(kind, router_mesh_port)?; + for input in routed_inputs { + let protocol = dynamic_input_route_mesh_protocol(input)?; + let component_target = mesh_plan + .targets + .iter_mut() + .find(|target| { + matches!(target.kind, MeshProvisionTargetKind::Component) + && target.config.identity.id == input.component + }) + .ok_or_else(|| { + miette::miette!( + "dynamic routed input {}.{} is missing component {} in the mesh provision plan", + input.component, + input.slot, + input.component + ) + })?; + if !component_target + .config + .peers + .iter() + .any(|peer| peer.id == router_identity_id) + { + component_target + .config + .peers + .push(amber_mesh::MeshPeerTemplate { + id: router_identity_id.to_string(), + }); + } + + let route_id = dynamic_input_route_route_id(input, protocol); + let capability = dynamic_input_route_capability(input); + let mut matched = false; + for route in component_target + .config + .outbound + .iter_mut() + .filter(|route| route.slot == input.slot) + { + matched = true; + route.route_id = route_id.clone(); + route.protocol = protocol; + route.peer_addr = router_peer_addr.clone(); + route.peer_id = router_identity_id.to_string(); + route.capability = capability.clone(); + route.capability_kind = Some(input.capability_kind.clone()); + route.capability_profile = input.capability_profile.clone(); + route.http_plugins = dynamic_input_route_http_plugins(input, protocol); + } + if !matched { + return Err(miette::miette!( + "dynamic routed input {}.{} is missing an outbound route in the mesh provision \ + plan", + input.component, + input.slot + )); + } + } + + Ok(()) +} + +pub(super) fn rewrite_dynamic_direct_inputs( + mesh_plan: &mut MeshProvisionPlan, + direct_inputs: &[DynamicInputDirectRecord], + live_components: &BTreeMap, +) -> Result<()> { + if direct_inputs.is_empty() { + return Ok(()); + } + + for input in direct_inputs { + let provider_runtime = live_components + .get(&input.provider_component) + .ok_or_else(|| { + miette::miette!( + "dynamic direct input {}.{} references live provider {} that is unavailable", + input.component, + input.slot, + input.provider_component + ) + })?; + let protocol = dynamic_input_direct_mesh_protocol(input)?; + let component_target = mesh_plan + .targets + .iter_mut() + .find(|target| { + matches!(target.kind, MeshProvisionTargetKind::Component) + && target.config.identity.id == input.component + }) + .ok_or_else(|| { + miette::miette!( + "dynamic direct input {}.{} is missing component {} in the mesh provision plan", + input.component, + input.slot, + input.component + ) + })?; + if !component_target + .config + .peers + .iter() + .any(|peer| peer.id == provider_runtime.mesh_config.identity.id) + { + component_target + .config + .peers + .push(amber_mesh::MeshPeerTemplate { + id: provider_runtime.mesh_config.identity.id.clone(), + }); + } + + let route_id = dynamic_input_direct_route_id(input, protocol); + let capability = dynamic_input_direct_capability(input); + let mut matched = false; + for route in component_target + .config + .outbound + .iter_mut() + .filter(|route| route.slot == input.slot) + { + matched = true; + route.route_id = route_id.clone(); + route.protocol = protocol; + route.peer_addr = provider_runtime.host_mesh_addr.clone(); + route.peer_id = provider_runtime.mesh_config.identity.id.clone(); + route.capability = capability.clone(); + route.capability_kind = Some(input.capability_kind.clone()); + route.capability_profile = input.capability_profile.clone(); + route.http_plugins = dynamic_input_direct_http_plugins(input, protocol); + } + if !matched { + return Err(miette::miette!( + "dynamic direct input {}.{} is missing an outbound route in the mesh provision \ + plan", + input.component, + input.slot + )); + } + } + + Ok(()) +} + +pub(super) fn rewrite_dynamic_routed_inputs_in_artifact( + artifact_root: &Path, + routed_inputs: &[DynamicInputRouteRecord], + kind: SiteKind, + router_identity_id: &str, + router_mesh_port: Option, +) -> Result<()> { + if routed_inputs.is_empty() { + return Ok(()); + } + let path = artifact_root.join("mesh-provision-plan.json"); + let mut mesh_plan: MeshProvisionPlan = read_json(&path, "mesh provision plan")?; + rewrite_dynamic_routed_inputs( + &mut mesh_plan, + routed_inputs, + kind, + router_identity_id, + router_mesh_port, + )?; + write_json(&path, &mesh_plan) +} + +pub(super) fn rewrite_dynamic_direct_inputs_in_artifact( + artifact_root: &Path, + direct_inputs: &[DynamicInputDirectRecord], + live_components: &BTreeMap, +) -> Result<()> { + if direct_inputs.is_empty() { + return Ok(()); + } + let path = artifact_root.join("mesh-provision-plan.json"); + let mut mesh_plan: MeshProvisionPlan = read_json(&path, "mesh provision plan")?; + rewrite_dynamic_direct_inputs(&mut mesh_plan, direct_inputs, live_components)?; + write_json(&path, &mesh_plan) +} + +pub(super) fn build_filesystem_route_overlay_base( + artifact_root: &Path, + assigned_components: &[String], + provider_peer_addrs: &BTreeMap, + peer_identities: &BTreeMap, +) -> Result { + let mesh_plan: MeshProvisionPlan = read_json( + &artifact_root.join("mesh-provision-plan.json"), + "mesh provision plan", + )?; + let kept_component_ids = assigned_components.iter().cloned().collect::>(); + let mut router_target = mesh_plan + .targets + .iter() + .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) + .cloned() + .ok_or_else(|| { + miette::miette!( + "dynamic artifact {} is missing a router mesh target", + artifact_root.display() + ) + })?; + filter_dynamic_router_target(&mut router_target, &kept_component_ids); + for route in &mut router_target.config.inbound { + if let InboundTarget::MeshForward { + peer_id, peer_addr, .. + } = &mut route.target + && let Some(resolved) = provider_peer_addrs.get(peer_id) + { + *peer_addr = resolved.clone(); + } + } + let peers = router_target + .config + .peers + .iter() + .map(|peer| { + peer_identities.get(&peer.id).map(|identity| MeshPeer { + id: identity.id.clone(), + public_key: identity.public_key, + }) + }) + .collect::>>() + .ok_or_else(|| { + miette::miette!( + "dynamic artifact {} is missing a live mesh identity for one of its route peers", + artifact_root.display() + ) + })?; + Ok(StoredRouteOverlayPayload { + peers, + inbound_routes: router_target.config.inbound, + }) +} + +pub(super) fn write_direct_vm_startup_route_overlay_payload( + artifact_root: &Path, + _artifact_kind: &str, + routed_inputs: &[DynamicInputRouteRecord], + provider_peer_addrs: &BTreeMap, + existing_site_peer_identities: &BTreeMap, +) -> Result<()> { + if routed_inputs.is_empty() { + return Ok(()); + } + let allowed_issuers = overlay_issuer_sets(routed_inputs)?; + let mut payload = StoredRouteOverlayPayload { + peers: Vec::new(), + inbound_routes: Vec::new(), + }; + augment_route_overlay_payload( + &mut payload, + &BTreeMap::new(), + routed_inputs, + provider_peer_addrs, + existing_site_peer_identities, + Some(&allowed_issuers), + true, + )?; + if payload.inbound_routes.is_empty() { + return Ok(()); + } + write_dynamic_route_overlay_payload(artifact_root, &payload) +} + +pub(super) fn write_direct_vm_live_route_overlay_payload( + artifact_root: &Path, + assigned_components: &[String], + proxy_exports: &BTreeMap, + routed_inputs: &[DynamicInputRouteRecord], + provider_peer_addrs: &BTreeMap, + peer_identities: &BTreeMap, +) -> Result<()> { + let mut payload = build_filesystem_route_overlay_base( + artifact_root, + assigned_components, + provider_peer_addrs, + peer_identities, + )?; + augment_route_overlay_payload( + &mut payload, + proxy_exports, + routed_inputs, + provider_peer_addrs, + peer_identities, + None, + false, + )?; + write_dynamic_route_overlay_payload(artifact_root, &payload) +} + +pub(super) fn ensure_dynamic_proxy_export_component_routes( + mesh_plan: &mut MeshProvisionPlan, + proxy_exports: &BTreeMap, + router_identity_id: &str, +) -> Result<()> { + for export in proxy_exports.values() { + let protocol = dynamic_proxy_export_mesh_protocol(export)?; + let route_id = component_route_id(&export.component, &export.provide, protocol); + let component_target = mesh_plan + .targets + .iter_mut() + .find(|target| { + matches!(target.kind, MeshProvisionTargetKind::Component) + && target.config.identity.id == export.component + }) + .ok_or_else(|| { + miette::miette!( + "dynamic proxy export provider {} is missing from the mesh provision plan", + export.component + ) + })?; + if !component_target + .config + .peers + .iter() + .any(|peer| peer.id == router_identity_id) + { + component_target + .config + .peers + .push(amber_mesh::MeshPeerTemplate { + id: router_identity_id.to_string(), + }); + } + if let Some(route) = component_target + .config + .inbound + .iter_mut() + .find(|route| route.route_id == route_id) + { + if !route + .allowed_issuers + .iter() + .any(|issuer| issuer == router_identity_id) + { + route.allowed_issuers.push(router_identity_id.to_string()); + route.allowed_issuers.sort(); + route.allowed_issuers.dedup(); + } + continue; + } + component_target.config.inbound.push(InboundRoute { + route_id, + capability: export.provide.clone(), + capability_kind: Some(export.capability_kind.clone()), + capability_profile: export.capability_profile.clone(), + protocol, + http_plugins: dynamic_proxy_export_http_plugins(export, protocol), + target: InboundTarget::Local { + port: export.target_port, + }, + allowed_issuers: vec![router_identity_id.to_string()], + }); + } + Ok(()) +} + +pub(super) fn ensure_dynamic_proxy_export_component_routes_in_artifact( + artifact_root: &Path, + proxy_exports: &BTreeMap, + router_identity_id: &str, +) -> Result<()> { + let plan_path = artifact_root.join("mesh-provision-plan.json"); + let mut mesh_plan: MeshProvisionPlan = read_json(&plan_path, "mesh provision plan")?; + ensure_dynamic_proxy_export_component_routes( + &mut mesh_plan, + proxy_exports, + router_identity_id, + )?; + write_json(&plan_path, &mesh_plan) +} + +pub(super) fn add_dynamic_proxy_export_overlay_routes( + inbound_routes: &mut Vec, + proxy_exports: &BTreeMap, + mut peer_addr_for_export: impl FnMut(&DynamicProxyExportRecord) -> Result, +) -> Result<()> { + for (export_name, export) in proxy_exports { + let protocol = dynamic_proxy_export_mesh_protocol(export)?; + let route_id = router_dynamic_export_route_id(&export.component, export_name, protocol); + let route = InboundRoute { + route_id, + capability: export_name.clone(), + capability_kind: Some(export.capability_kind.clone()), + capability_profile: export.capability_profile.clone(), + protocol, + http_plugins: dynamic_proxy_export_http_plugins(export, protocol), + target: InboundTarget::MeshForward { + peer_addr: peer_addr_for_export(export)?, + peer_id: export.component.clone(), + route_id: component_route_id(&export.component, &export.provide, protocol), + capability: export.provide.clone(), + }, + allowed_issuers: Vec::new(), + }; + if let Some(existing) = inbound_routes + .iter_mut() + .find(|existing| existing.route_id == route.route_id) + { + *existing = route; + } else { + inbound_routes.push(route); + } + } + Ok(()) +} + +pub(super) struct DynamicComposeMeshPlan { + pub(super) mesh_plan: MeshProvisionPlan, + pub(super) mesh_dirs: BTreeMap, + pub(super) component_mesh_dirs: BTreeMap, +} + +pub(super) fn build_dynamic_compose_mesh_plan( + artifact_root: &Path, + assigned_components: &[String], +) -> Result { + let plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; + let assigned = assigned_components + .iter() + .map(String::as_str) + .collect::>(); + let mut filtered_targets = Vec::new(); + let mut mesh_dirs = BTreeMap::new(); + let mut component_mesh_dirs = BTreeMap::new(); + + for mut target in plan.targets { + match target.kind { + MeshProvisionTargetKind::Component => { + if !assigned.contains(target.config.identity.id.as_str()) { + continue; + } + let MeshProvisionOutput::Filesystem { dir } = &mut target.output else { + return Err(miette::miette!( + "compose component {} does not use filesystem mesh output", + target.config.identity.id + )); + }; + let sidecar = Path::new(dir.as_str()) + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| { + miette::miette!( + "compose component {} has invalid mesh output dir {}", + target.config.identity.id, + dir + ) + })? + .to_string(); + let relative_dir = dynamic_compose_mesh_dir(&sidecar); + dir.clear(); + dir.push_str(&relative_dir); + component_mesh_dirs.insert(target.config.identity.id.clone(), relative_dir.clone()); + mesh_dirs.insert(sidecar, relative_dir); + filtered_targets.push(target); + } + MeshProvisionTargetKind::Router => {} + } + } + + if filtered_targets.is_empty() { + return Err(miette::miette!( + "compose child artifact {} does not contain assigned child mesh targets", + artifact_root.join("compose.yaml").display() + )); + } + + Ok(DynamicComposeMeshPlan { + mesh_plan: MeshProvisionPlan { + version: plan.version, + identity_seed: plan.identity_seed, + existing_peer_identities: Vec::new(), + targets: filtered_targets, + }, + mesh_dirs, + component_mesh_dirs, + }) +} + +pub(super) fn project_dynamic_direct_router_surface( + plan: &SiteControllerRuntimePlan, + child: &SiteControllerRuntimeChildRecord, +) -> Result<()> { + let state_path = direct_runtime_state_path(Path::new(&child.artifact_root)); + let router_mesh_port = plan.router_mesh_port.ok_or_else(|| { + miette::miette!( + "direct site `{}` is missing its router mesh port", + plan.site_id + ) + })?; + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let mut state: DirectRuntimeState = read_json(&state_path, "direct runtime state")?; + if state.router_mesh_port == Some(router_mesh_port) { + break; + } + state.router_mesh_port = Some(router_mesh_port); + write_json(&state_path, &state)?; + std::thread::sleep(Duration::from_millis(100)); + let state: DirectRuntimeState = read_json(&state_path, "direct runtime state")?; + if state.router_mesh_port == Some(router_mesh_port) { + break; + } + if std::time::Instant::now() >= deadline { + return Err(miette::miette!( + "timed out projecting direct child router mesh port into {}", + state_path.display() + )); + } + } + + let site_current = direct_current_control_socket_path(Path::new(&plan.artifact_dir)); + let child_current = direct_current_control_socket_path(Path::new(&child.artifact_root)); + if let Some(parent) = child_current.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + ensure_direct_control_socket_link( + &child_current, + &site_current, + "dynamic child router control symlink", + )?; + let artifact_link = Path::new(&child.artifact_root) + .join(".amber") + .join("control") + .join("router-control.sock"); + if let Some(parent) = artifact_link.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + ensure_direct_control_socket_link( + &artifact_link, + &child_current, + "dynamic child router control artifact symlink", + ) +} + +pub(super) fn project_dynamic_vm_router_surface( + plan: &SiteControllerRuntimePlan, + child: &SiteControllerRuntimeChildRecord, +) -> Result<()> { + let state_path = Path::new(&child.artifact_root) + .join(".amber") + .join("vm-runtime.json"); + let runtime_root = site_controller_runtime_child_runtime_root(plan, child.child_id); + let router_mesh_port = plan.router_mesh_port.ok_or_else(|| { + miette::miette!("vm site `{}` is missing its router mesh port", plan.site_id) + })?; + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let mut state = + load_vm_runtime_state_for_artifact(Path::new(&child.artifact_root), &runtime_root)?; + if state.router_mesh_port == Some(router_mesh_port) { + break; + } + state.router_mesh_port = Some(router_mesh_port); + write_vm_runtime_state(Path::new(&child.artifact_root), &state)?; + std::thread::sleep(Duration::from_millis(100)); + let state = + load_vm_runtime_state_for_artifact(Path::new(&child.artifact_root), &runtime_root)?; + if state.router_mesh_port == Some(router_mesh_port) { + break; + } + if std::time::Instant::now() >= deadline { + return Err(miette::miette!( + "timed out projecting vm child router mesh port into {}", + state_path.display() + )); + } + } + + let site_current = vm_current_control_socket_path(Path::new(&plan.artifact_dir)); + let child_current = vm_current_control_socket_path(Path::new(&child.artifact_root)); + if let Some(parent) = child_current.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + ensure_control_socket_link( + &child_current, + &site_current, + "dynamic child vm router control symlink", + )?; + let artifact_link = Path::new(&child.artifact_root) + .join(".amber") + .join("control") + .join("router-control.sock"); + if let Some(parent) = artifact_link.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + ensure_control_socket_link( + &artifact_link, + &child_current, + "dynamic child vm router control artifact symlink", + ) +} + +pub(super) fn dynamic_child_route_overlay_id( + plan: &SiteControllerRuntimePlan, + child_id: u64, +) -> String { + format!("framework-child:{}:{child_id}", plan.site_id) +} + +pub(super) fn site_router_control_endpoint( + plan: &SiteControllerRuntimePlan, +) -> Result { + let state_path = Path::new(&plan.site_state_root).join("manager-state.json"); + if state_path.is_file() { + let state: SiteManagerState = read_json(&state_path, "site manager state")?; + if let Some(raw) = state.router_control { + return parse_control_endpoint(&raw); + } + } + if let Some(raw) = plan.local_router_control.as_deref() { + return parse_control_endpoint(raw); + } + + match plan.kind { + SiteKind::Direct => Ok(ControlEndpoint::Unix(direct_current_control_socket_path( + Path::new(&plan.artifact_dir), + ))), + SiteKind::Vm => Ok(ControlEndpoint::Unix(vm_current_control_socket_path( + Path::new(&plan.artifact_dir), + ))), + SiteKind::Compose | SiteKind::Kubernetes => Err(miette::miette!( + "site `{}` manager state is missing router control endpoint", + plan.site_id + )), + } +} + +pub(super) fn child_router_overlay_payload( + plan: &SiteControllerRuntimePlan, + artifact_root: &Path, + runtime_root: &Path, +) -> Result<(Vec, Vec)> { + let overlay_path = dynamic_route_overlay_path(artifact_root); + if overlay_path.is_file() { + let payload: StoredRouteOverlayPayload = read_json(&overlay_path, "site router overlay")?; + return Ok((payload.peers, payload.inbound_routes)); + } + let provision: MeshProvisionPlan = read_json( + &artifact_root.join("mesh-provision-plan.json"), + "mesh provision plan", + )?; + let Some(router_target) = provision + .targets + .iter() + .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) + else { + return Ok((Vec::new(), Vec::new())); + }; + let proxy_exports = load_dynamic_proxy_exports_metadata(artifact_root)?; + let mut overlay_peer_ids = router_target + .config + .peers + .iter() + .map(|peer| peer.id.clone()) + .collect::>(); + overlay_peer_ids.extend( + proxy_exports + .values() + .map(|export| export.component.clone()), + ); + + let mut peers = Vec::new(); + let mut peer_addr_by_id = BTreeMap::new(); + for peer_id in &overlay_peer_ids { + let component_target = provision + .targets + .iter() + .find(|target| { + matches!(target.kind, MeshProvisionTargetKind::Component) + && target.config.identity.id == *peer_id + }) + .ok_or_else(|| { + miette::miette!( + "router overlay peer {} is missing from mesh provision plan", + peer_id + ) + })?; + let (identity, runtime_config) = match &component_target.output { + MeshProvisionOutput::Filesystem { dir } => { + let identity: MeshIdentitySecret = read_json( + &runtime_root.join(dir).join(MESH_IDENTITY_FILENAME), + "mesh identity", + )?; + let runtime_config: MeshConfigPublic = read_json( + &runtime_root.join(dir).join(MESH_CONFIG_FILENAME), + "mesh config", + )?; + (identity, Some(runtime_config)) + } + MeshProvisionOutput::KubernetesSecret { name, namespace } => ( + load_kubernetes_mesh_identity_secret(plan, name, namespace.as_deref())?, + None, + ), + }; + let public_key = identity.public_key().into_diagnostic()?; + if let Some(runtime_config) = runtime_config { + peer_addr_by_id.insert( + runtime_config.identity.id.clone(), + runtime_config.mesh_listen.to_string(), + ); + } + peers.push(MeshPeer { + id: identity.id, + public_key, + }); + } + let mut inbound_routes = router_target.config.inbound.clone(); + for route in &mut inbound_routes { + if let InboundTarget::MeshForward { + peer_id, peer_addr, .. + } = &mut route.target + && let Some(resolved) = peer_addr_by_id.get(peer_id) + { + peer_addr.clone_from(resolved); + } + } + if !proxy_exports.is_empty() { + add_dynamic_proxy_export_overlay_routes(&mut inbound_routes, &proxy_exports, |export| { + match plan.kind { + SiteKind::Kubernetes => { + dynamic_proxy_export_kubernetes_peer_addr(artifact_root, &provision, export) + } + SiteKind::Direct | SiteKind::Vm | SiteKind::Compose => peer_addr_by_id + .get(&export.component) + .map(ToString::to_string) + .ok_or_else(|| { + miette::miette!( + "dynamic proxy export provider {} is missing a live mesh address", + export.component + ) + }), + } + })?; + } + Ok((peers, inbound_routes)) +} + +pub(super) fn child_overlay_runtime_root( + plan: &SiteControllerRuntimePlan, + child: &SiteControllerRuntimeChildRecord, +) -> PathBuf { + match plan.kind { + SiteKind::Direct | SiteKind::Vm => { + site_controller_runtime_child_runtime_root(plan, child.child_id) + } + SiteKind::Compose | SiteKind::Kubernetes => PathBuf::from(&child.artifact_root), + } +} + +pub(super) fn dynamic_route_issuer_grants( + children: &[SiteControllerRuntimeChildRecord], +) -> Result>> { + let mut issuers_by_route_id = BTreeMap::>::new(); + for child in children { + for input in &child.routed_inputs { + let route_id = + dynamic_input_route_route_id(input, dynamic_input_route_mesh_protocol(input)?); + issuers_by_route_id + .entry(route_id) + .or_default() + .insert(input.component.clone()); + } + } + Ok(issuers_by_route_id) +} + +pub(super) fn load_published_component_peers( + plan: &SiteControllerRuntimePlan, + published_children: &[SiteControllerRuntimeChildRecord], +) -> Result> { + let mut component_peers = BTreeMap::new(); + for child in published_children { + if child.assigned_components.is_empty() { + continue; + } + let artifact_root = Path::new(&child.artifact_root); + let runtime_root = child_overlay_runtime_root(plan, child); + let provision: MeshProvisionPlan = read_json( + &artifact_root.join("mesh-provision-plan.json"), + "mesh provision plan", + )?; + for component in &child.assigned_components { + let target = provision + .targets + .iter() + .find(|target| { + matches!(target.kind, MeshProvisionTargetKind::Component) + && target.config.identity.id == *component + }) + .ok_or_else(|| { + miette::miette!( + "published child {} is missing component {} in its mesh provision plan", + child.child_id, + component + ) + })?; + let identity = match &target.output { + MeshProvisionOutput::Filesystem { dir } => read_json( + &runtime_root.join(dir).join(MESH_IDENTITY_FILENAME), + "mesh identity", + )?, + MeshProvisionOutput::KubernetesSecret { name, namespace } => { + load_kubernetes_mesh_identity_secret(plan, name, namespace.as_deref())? + } + }; + component_peers.insert( + component.clone(), + MeshPeer { + id: identity.id.clone(), + public_key: identity.public_key().into_diagnostic()?, + }, + ); + } + } + Ok(component_peers) +} + +pub(super) fn apply_dynamic_route_issuer_grants( + peers: &mut Vec, + inbound_routes: &mut [InboundRoute], + issuers_by_route_id: &BTreeMap>, + component_peers: &BTreeMap, +) -> Result<()> { + let mut known_peer_ids = peers + .iter() + .map(|peer| peer.id.clone()) + .collect::>(); + for route in inbound_routes { + let Some(issuers) = issuers_by_route_id.get(&route.route_id) else { + continue; + }; + route.allowed_issuers = issuers.iter().cloned().collect(); + for issuer in issuers { + if known_peer_ids.contains(issuer) { + continue; + } + let peer = component_peers.get(issuer).ok_or_else(|| { + miette::miette!( + "dynamic route {} references published issuer {} with no live mesh peer", + route.route_id, + issuer + ) + })?; + peers.push(peer.clone()); + known_peer_ids.insert(issuer.clone()); + } + } + Ok(()) +} + +pub(super) fn dynamic_direct_input_overlay_id(component: &str) -> String { + format!("framework-direct-inputs:{component}") +} + +pub(super) fn dynamic_direct_input_grants( + children: &[SiteControllerRuntimeChildRecord], +) -> Result>>> { + let mut grants = BTreeMap::>>::new(); + for child in children { + for input in &child.direct_inputs { + let route_id = + dynamic_input_direct_route_id(input, dynamic_input_direct_mesh_protocol(input)?); + grants + .entry(input.provider_component.clone()) + .or_default() + .entry(route_id) + .or_default() + .insert(input.component.clone()); + } + } + Ok(grants) +} + +pub(super) async fn reconcile_dynamic_direct_input_overlays( + app: &SiteControllerRuntimeApp, +) -> Result<()> { + let published_children = { + let state = app.state.lock().await; + state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>() + }; + let live_components = collect_live_component_runtime_metadata(&app.plan)?; + let grants = dynamic_direct_input_grants(&published_children)?; + for (component, runtime) in &live_components { + let Some(control_endpoint) = runtime.control_endpoint.as_ref() else { + continue; + }; + let overlay_id = dynamic_direct_input_overlay_id(component); + let Some(route_grants) = grants.get(component) else { + revoke_route_overlay_with_retry(control_endpoint, &overlay_id, Duration::from_secs(30)) + .await?; + continue; + }; + + let mut peers = Vec::new(); + let mut known_peer_ids = BTreeSet::new(); + let mut inbound_routes = Vec::new(); + for (route_id, issuers) in route_grants { + let base_route = runtime + .mesh_config + .inbound + .iter() + .find(|route| route.route_id == *route_id) + .cloned() + .ok_or_else(|| { + miette::miette!( + "provider component {} is missing inbound route {} for a direct dynamic \ + input", + component, + route_id + ) + })?; + let mut route = base_route; + route.allowed_issuers = issuers.iter().cloned().collect(); + for issuer in issuers { + let issuer_runtime = live_components.get(issuer).ok_or_else(|| { + miette::miette!( + "direct dynamic input issuer {} is not live on site {}", + issuer, + app.plan.site_id + ) + })?; + if known_peer_ids.insert(issuer_runtime.mesh_config.identity.id.clone()) { + peers.push(MeshPeer { + id: issuer_runtime.mesh_config.identity.id.clone(), + public_key: issuer_runtime.mesh_config.identity.public_key, + }); + } + } + inbound_routes.push(route); + } + apply_route_overlay_with_retry( + control_endpoint, + &overlay_id, + &peers, + &inbound_routes, + Duration::from_secs(30), + ) + .await?; + } + Ok(()) +} + +pub(super) async fn reconcile_dynamic_site_router_overlays( + app: &SiteControllerRuntimeApp, +) -> Result<()> { + let published_children = { + let state = app.state.lock().await; + state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>() + }; + reconcile_dynamic_site_router_overlays_for_children( + app, + &published_children, + &published_children, + ) + .await +} + +pub(super) async fn reconcile_dynamic_site_router_overlays_for_children( + app: &SiteControllerRuntimeApp, + overlay_children: &[SiteControllerRuntimeChildRecord], + issuer_children: &[SiteControllerRuntimeChildRecord], +) -> Result<()> { + if overlay_children.is_empty() { + return Ok(()); + } + + let issuers_by_route_id = dynamic_route_issuer_grants(issuer_children)?; + let component_peers = load_published_component_peers(&app.plan, issuer_children)?; + for child in overlay_children { + let artifact_root = Path::new(&child.artifact_root); + let runtime_root = child_overlay_runtime_root(&app.plan, child); + let (mut peers, mut inbound_routes) = + child_router_overlay_payload(&app.plan, artifact_root, &runtime_root)?; + apply_dynamic_route_issuer_grants( + &mut peers, + &mut inbound_routes, + &issuers_by_route_id, + &component_peers, + )?; + if inbound_routes.is_empty() { + continue; + } + let endpoint = site_router_control_endpoint(&app.plan)?; + apply_route_overlay_with_retry( + &endpoint, + &dynamic_child_route_overlay_id(&app.plan, child.child_id), + &peers, + &inbound_routes, + Duration::from_secs(30), + ) + .await?; + } + + Ok(()) +} + +pub(super) async fn apply_dynamic_site_router_overlay( + plan: &SiteControllerRuntimePlan, + child: &SiteControllerRuntimeChildRecord, +) -> Result<()> { + let artifact_root = Path::new(&child.artifact_root); + let runtime_root = child_overlay_runtime_root(plan, child); + let (peers, inbound_routes) = child_router_overlay_payload(plan, artifact_root, &runtime_root)?; + if inbound_routes.is_empty() { + return Ok(()); + } + let endpoint = site_router_control_endpoint(plan)?; + apply_route_overlay_with_retry( + &endpoint, + &dynamic_child_route_overlay_id(plan, child.child_id), + &peers, + &inbound_routes, + Duration::from_secs(30), + ) + .await +} + +pub(super) async fn revoke_dynamic_site_router_overlay( + plan: &SiteControllerRuntimePlan, + child: &SiteControllerRuntimeChildRecord, +) -> Result<()> { + let endpoint = site_router_control_endpoint(plan)?; + revoke_route_overlay_with_retry( + &endpoint, + &dynamic_child_route_overlay_id(plan, child.child_id), + Duration::from_secs(30), + ) + .await +} + +#[cfg(test)] +mod direct_input_tests { + use std::fs; + + use amber_mesh::MeshConfigTemplate; + + use super::*; + + #[test] + fn rewrite_dynamic_direct_inputs_points_child_at_provider_sidecar() { + let temp = tempfile::tempdir().expect("tempdir"); + let path = temp.path().join("mesh-provision-plan.json"); + write_json( + &path, + &MeshProvisionPlan { + version: amber_mesh::MESH_PROVISION_PLAN_VERSION.to_string(), + identity_seed: None, + existing_peer_identities: Vec::new(), + targets: vec![MeshProvisionTarget { + kind: MeshProvisionTargetKind::Component, + config: MeshConfigTemplate { + identity: amber_mesh::MeshIdentityTemplate { + id: "/consumer".to_string(), + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:0".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: vec![OutboundRoute { + route_id: "old".to_string(), + slot: "api".to_string(), + capability_kind: Some("http".to_string()), + capability_profile: None, + listen_port: 0, + listen_addr: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + peer_addr: "127.0.0.1:1".to_string(), + peer_id: "/router".to_string(), + capability: "old".to_string(), + }], + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + output: MeshProvisionOutput::Filesystem { + dir: "mesh/components/consumer".to_string(), + }, + }], + }, + ) + .expect("write mesh plan"); + + let mut live_components = BTreeMap::new(); + live_components.insert( + "/provider".to_string(), + LiveComponentRuntimeMetadata { + moniker: "/provider".to_string(), + host_mesh_addr: "10.0.0.20:24001".to_string(), + control_endpoint: None, + mesh_config: MeshConfigPublic { + identity: MeshIdentityPublic { + id: "/provider".to_string(), + public_key: [7; 32], + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:0".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + }, + ); + rewrite_dynamic_direct_inputs_in_artifact( + temp.path(), + &[DynamicInputDirectRecord { + component: "/consumer".to_string(), + slot: "api".to_string(), + provider_component: "/provider".to_string(), + protocol: "http".to_string(), + capability_kind: "http".to_string(), + capability_profile: None, + target: DynamicInputRouteTarget::ComponentProvide { + provide: "serve".to_string(), + }, + }], + &live_components, + ) + .expect("rewrite should succeed"); + + let plan: MeshProvisionPlan = read_json(&path, "mesh provision plan").expect("plan"); + let target = &plan.targets[0]; + assert_eq!(target.config.peers.len(), 1); + assert_eq!(target.config.peers[0].id, "/provider"); + let route = &target.config.outbound[0]; + assert_eq!(route.peer_id, "/provider"); + assert_eq!(route.peer_addr, "10.0.0.20:24001"); + assert_eq!(route.capability, "serve"); + assert_eq!( + route.route_id, + component_route_id("/provider", "serve", MeshProtocol::Http) + ); + } + + #[test] + fn dynamic_route_issuer_grants_include_component_provide_inputs() { + let issuers = dynamic_route_issuer_grants(&[SiteControllerRuntimeChildRecord { + child_id: 7, + artifact_root: "/tmp/child".to_string(), + assigned_components: vec!["/sibling".to_string()], + proxy_exports: BTreeMap::new(), + direct_inputs: Vec::new(), + routed_inputs: vec![DynamicInputRouteRecord { + component: "/sibling".to_string(), + slot: "upstream".to_string(), + provider_component: "/provider".to_string(), + protocol: "http".to_string(), + capability_kind: "http".to_string(), + capability_profile: None, + target: DynamicInputRouteTarget::ComponentProvide { + provide: "http".to_string(), + }, + }], + process_pid: None, + published: true, + }]) + .expect("component-provide routed inputs should produce issuer grants"); + + assert_eq!( + issuers.get("component:/provider:http:http"), + Some(&BTreeSet::from(["/sibling".to_string()])) + ); + } + + #[test] + fn cleanup_dynamic_site_children_removes_child_roots_and_clears_state() { + let temp = tempfile::tempdir().expect("tempdir should be created"); + let site_state_root = temp.path().join("state").join("direct_local"); + let child_root = + runtime_api::site_controller_runtime_child_root_for_site(&site_state_root, 7); + fs::create_dir_all(child_root.join("artifact")).expect("child artifact dir should exist"); + fs::write(child_root.join("artifact").join("marker.txt"), "marker") + .expect("child marker should be written"); + write_json( + &site_controller_runtime_state_path(&site_state_root), + &SiteControllerRuntimeState { + schema: SITE_CONTROLLER_RUNTIME_STATE_SCHEMA.to_string(), + version: SITE_CONTROLLER_RUNTIME_STATE_VERSION, + run_id: "run-123".to_string(), + site_id: "direct_local".to_string(), + kind: SiteKind::Direct, + children: BTreeMap::from([( + 7, + SiteControllerRuntimeChildRecord { + child_id: 7, + artifact_root: child_root.join("artifact").display().to_string(), + assigned_components: Vec::new(), + proxy_exports: BTreeMap::new(), + direct_inputs: Vec::new(), + routed_inputs: Vec::new(), + process_pid: None, + published: true, + }, + )]), + }, + ) + .expect("site controller runtime state should be written"); + + cleanup_dynamic_site_children(&site_state_root, SiteKind::Direct) + .expect("dynamic site children should be cleaned"); + + let state: SiteControllerRuntimeState = read_json( + &site_controller_runtime_state_path(&site_state_root), + "site controller runtime state", + ) + .expect("site controller runtime state should be readable"); + assert!(state.children.is_empty()); + assert!(!child_root.exists()); + } + + #[test] + fn inject_site_controller_peer_router_routes_adds_local_inbound_and_peer_outbound_routes() { + let temp = tempfile::tempdir().expect("tempdir"); + let path = temp.path().join("mesh-provision-plan.json"); + write_json( + &path, + &MeshProvisionPlan { + version: amber_mesh::MESH_PROVISION_PLAN_VERSION.to_string(), + identity_seed: None, + existing_peer_identities: Vec::new(), + targets: vec![MeshProvisionTarget { + kind: MeshProvisionTargetKind::Router, + config: MeshConfigTemplate { + identity: amber_mesh::MeshIdentityTemplate { + id: "/site/local/router".to_string(), + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:24000".parse().expect("mesh listen"), + control_listen: Some("127.0.0.1:24100".parse().expect("control listen")), + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + output: MeshProvisionOutput::Filesystem { + dir: "mesh/router".to_string(), + }, + }], + }, + ) + .expect("write mesh plan"); + + inject_site_controller_peer_router_routes( + temp.path(), + "local", + &["/site/peer/router".to_string()], + &[SiteControllerPeerRouterRoute { + site_id: "peer".to_string(), + peer_router: MeshIdentityPublic { + id: "/site/peer/router".to_string(), + public_key: [7; 32], + mesh_scope: Some("amber.test".to_string()), + }, + peer_addr: "10.0.0.20:24000".to_string(), + listen_addr: "127.0.0.1".to_string(), + listen_port: 25001, + }], + ) + .expect("inject routes"); + + let plan: MeshProvisionPlan = read_json(&path, "mesh provision plan").expect("plan"); + let router = &plan.targets[0]; + assert!( + router.config.inbound.iter().any(|route| { + route.route_id == site_controller_internal_route_id("local") + && route.capability == SITE_CONTROLLER_INTERNAL_CAPABILITY + && route.allowed_issuers == vec!["/site/peer/router".to_string()] + && matches!( + route.target, + InboundTarget::External { ref url_env, optional } + if url_env == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV + && !optional + ) + }), + "router should expose a local inbound route for peer site-controller traffic", + ); + assert!( + router.config.outbound.iter().any(|route| { + route.route_id == site_controller_internal_route_id("peer") + && route.capability == SITE_CONTROLLER_INTERNAL_CAPABILITY + && route.peer_id == "/site/peer/router" + && route.peer_addr == "10.0.0.20:24000" + && route.listen_addr.as_deref() == Some("127.0.0.1") + && route.listen_port == 25001 + }), + "router should publish a local outbound listener for the peer site controller", + ); + assert!( + plan.existing_peer_identities.iter().any(|identity| { + identity.id == "/site/peer/router" + && identity.public_key == [7; 32] + && identity.mesh_scope.as_deref() == Some("amber.test") + }), + "router mesh plan should carry the peer router identity needed to provision the route", + ); + } + + #[test] + fn compose_router_port_helpers_rewrite_mesh_publish_and_add_controller_ports() { + let temp = tempfile::tempdir().expect("tempdir"); + fs::write( + temp.path().join("compose.yaml"), + r#" +services: + amber-router: + image: ghcr.io/rdi-foundation/amber-router:v0.1.x + ports: + - "127.0.0.1::24000" +"#, + ) + .expect("compose yaml should write"); + + set_compose_router_published_mesh_port(temp.path(), 34000).expect("rewrite mesh port"); + add_compose_router_published_route_ports(temp.path(), &[34001, 34002]) + .expect("append controller ports"); + + let rendered = + fs::read_to_string(temp.path().join("compose.yaml")).expect("compose yaml should read"); + assert!( + rendered.contains("0.0.0.0:34000:24000"), + "mesh port should be rewritten to a deterministic host publish:\n{rendered}" + ); + assert!( + rendered.contains("127.0.0.1:34001:34001"), + "controller route port 34001 should be published on loopback:\n{rendered}" + ); + assert!( + rendered.contains("127.0.0.1:34002:34002"), + "controller route port 34002 should be published on loopback:\n{rendered}" + ); + } +} diff --git a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs new file mode 100644 index 00000000..90150e0a --- /dev/null +++ b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs @@ -0,0 +1,464 @@ +use std::{ + collections::BTreeMap, + fs, + io::Write as _, + net::SocketAddr, + path::{Path, PathBuf}, +}; + +use base64::Engine as _; +use serde_json::json; +use tar::Builder; + +use super::*; + +const KUBERNETES_ROUTER_NAME: &str = "amber-router"; +const KUBERNETES_ROUTER_MESH_PORT: u16 = 24000; +const KUBERNETES_ROUTER_CONTROL_PORT: u16 = 24100; +const KUBERNETES_ROUTER_NETPOL_PATH: &str = "05-networkpolicies/amber-router-netpol.yaml"; +const KUBERNETES_CONTROLLER_SEED_CONFIGMAP_PATH: &str = + "01-configmaps/amber-site-controller-seed.yaml"; +const KUBERNETES_CONTROLLER_SERVICE_ACCOUNT_PATH: &str = "02-rbac/amber-site-controller-sa.yaml"; +const KUBERNETES_CONTROLLER_ROLE_PATH: &str = "02-rbac/amber-site-controller-role.yaml"; +const KUBERNETES_CONTROLLER_ROLE_BINDING_PATH: &str = + "02-rbac/amber-site-controller-rolebinding.yaml"; +const KUBERNETES_CONTROLLER_DEPLOYMENT_PATH: &str = "03-deployments/amber-site-controller.yaml"; +const KUBERNETES_CONTROLLER_SERVICE_PATH: &str = "04-services/amber-site-controller.yaml"; +const KUBERNETES_CONTROLLER_SEED_VOLUME: &str = "controller-seed"; +const KUBERNETES_CONTROLLER_STATE_VOLUME: &str = "controller-state"; +const KUBERNETES_CONTROLLER_SITE_ROOT: &str = "/amber/site"; +const KUBERNETES_CONTROLLER_STATE_ROOT: &str = "/amber/site/state"; +const KUBERNETES_CONTROLLER_ARTIFACT_ROOT: &str = "/amber/site/artifact"; +const KUBERNETES_CONTROLLER_PLAN_PATH: &str = "/amber/site/state/site-controller-plan.json"; +const KUBERNETES_CONTROLLER_STATE_PATH: &str = "/amber/site/state/site-controller-state.json"; +const KUBERNETES_CONTROLLER_DESIRED_LINKS_PATH: &str = "/amber/site/state/desired-links.json"; +const KUBERNETES_CONTROLLER_SEED_ROOT: &str = "/amber/seed"; + +pub fn inject_kubernetes_site_controller( + artifact_root: &Path, + plan: &SiteControllerPlan, + controller_image: &str, +) -> Result<()> { + let embedded_plan = build_embedded_kubernetes_controller_plan(plan); + let seed_configmap = + build_kubernetes_controller_seed_configmap(artifact_root, plan, &embedded_plan)?; + let labels = kubernetes_controller_labels(); + let selector = kubernetes_controller_selector(); + + write_yaml_artifact( + artifact_root.join(KUBERNETES_CONTROLLER_SEED_CONFIGMAP_PATH), + &seed_configmap, + )?; + write_yaml_artifact( + artifact_root.join(KUBERNETES_CONTROLLER_SERVICE_ACCOUNT_PATH), + &json!({ + "apiVersion": "v1", + "kind": "ServiceAccount", + "metadata": { + "name": SITE_CONTROLLER_SERVICE_NAME, + "labels": labels, + } + }), + )?; + write_yaml_artifact( + artifact_root.join(KUBERNETES_CONTROLLER_ROLE_PATH), + &json!({ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "Role", + "metadata": { + "name": SITE_CONTROLLER_SERVICE_NAME, + "labels": labels, + }, + "rules": [ + { + "apiGroups": [""], + "resources": [ + "configmaps", + "persistentvolumeclaims", + "secrets", + "serviceaccounts", + "services" + ], + "verbs": ["create", "delete", "get", "list", "patch", "update", "watch"], + }, + { + "apiGroups": ["apps"], + "resources": ["deployments"], + "verbs": ["create", "delete", "get", "list", "patch", "update", "watch"], + }, + { + "apiGroups": ["batch"], + "resources": ["jobs"], + "verbs": ["create", "delete", "get", "list", "patch", "update", "watch"], + }, + { + "apiGroups": ["networking.k8s.io"], + "resources": ["networkpolicies"], + "verbs": ["create", "delete", "get", "list", "patch", "update", "watch"], + }, + { + "apiGroups": ["rbac.authorization.k8s.io"], + "resources": ["roles", "rolebindings"], + "verbs": ["create", "delete", "get", "list", "patch", "update", "watch"], + } + ] + }), + )?; + write_yaml_artifact( + artifact_root.join(KUBERNETES_CONTROLLER_ROLE_BINDING_PATH), + &json!({ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "RoleBinding", + "metadata": { + "name": SITE_CONTROLLER_SERVICE_NAME, + "labels": labels, + }, + "subjects": [{ + "kind": "ServiceAccount", + "name": SITE_CONTROLLER_SERVICE_NAME, + }], + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Role", + "name": SITE_CONTROLLER_SERVICE_NAME, + } + }), + )?; + write_yaml_artifact( + artifact_root.join(KUBERNETES_CONTROLLER_DEPLOYMENT_PATH), + &json!({ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": SITE_CONTROLLER_SERVICE_NAME, + "labels": labels, + }, + "spec": { + "replicas": 1, + "selector": { + "matchLabels": selector, + }, + "template": { + "metadata": { + "labels": labels, + }, + "spec": { + "serviceAccountName": SITE_CONTROLLER_SERVICE_NAME, + "automountServiceAccountToken": true, + "initContainers": [{ + "name": "seed-site-controller", + "image": "busybox:1.36.1", + "command": [ + "sh", + "-lc", + format!( + "set -eu\nmkdir -p {KUBERNETES_CONTROLLER_STATE_ROOT} \ + {KUBERNETES_CONTROLLER_ARTIFACT_ROOT}\ncp \ + {KUBERNETES_CONTROLLER_SEED_ROOT}/site-controller-plan.json \ + {KUBERNETES_CONTROLLER_PLAN_PATH}\ncp \ + {KUBERNETES_CONTROLLER_SEED_ROOT}/site-controller-state.json \ + {KUBERNETES_CONTROLLER_STATE_PATH}\ncp \ + {KUBERNETES_CONTROLLER_SEED_ROOT}/desired-links.json \ + {KUBERNETES_CONTROLLER_DESIRED_LINKS_PATH}\nbase64 -d \ + {KUBERNETES_CONTROLLER_SEED_ROOT}/artifact.tar.b64 | tar \ + -xf - -C {KUBERNETES_CONTROLLER_ARTIFACT_ROOT}\n" + ) + ], + "volumeMounts": [ + { + "name": KUBERNETES_CONTROLLER_SEED_VOLUME, + "mountPath": KUBERNETES_CONTROLLER_SEED_ROOT, + "readOnly": true, + }, + { + "name": KUBERNETES_CONTROLLER_STATE_VOLUME, + "mountPath": KUBERNETES_CONTROLLER_SITE_ROOT, + } + ] + }], + "containers": [{ + "name": SITE_CONTROLLER_SERVICE_NAME, + "image": controller_image, + "command": ["--plan", KUBERNETES_CONTROLLER_PLAN_PATH], + "ports": [{ + "name": "http", + "containerPort": SITE_CONTROLLER_PORT, + "protocol": "TCP", + }], + "volumeMounts": [{ + "name": KUBERNETES_CONTROLLER_STATE_VOLUME, + "mountPath": KUBERNETES_CONTROLLER_SITE_ROOT, + }] + }], + "volumes": [ + { + "name": KUBERNETES_CONTROLLER_SEED_VOLUME, + "configMap": { + "name": format!("{SITE_CONTROLLER_SERVICE_NAME}-seed"), + } + }, + { + "name": KUBERNETES_CONTROLLER_STATE_VOLUME, + "emptyDir": {} + } + ] + } + } + } + }), + )?; + write_yaml_artifact( + artifact_root.join(KUBERNETES_CONTROLLER_SERVICE_PATH), + &json!({ + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": SITE_CONTROLLER_SERVICE_NAME, + "labels": labels, + }, + "spec": { + "selector": selector, + "ports": [{ + "name": "http", + "port": SITE_CONTROLLER_PORT, + "targetPort": SITE_CONTROLLER_PORT, + "protocol": "TCP", + }], + "type": "ClusterIP", + } + }), + )?; + add_kubernetes_resource_paths( + artifact_root, + &[ + KUBERNETES_CONTROLLER_SEED_CONFIGMAP_PATH, + KUBERNETES_CONTROLLER_SERVICE_ACCOUNT_PATH, + KUBERNETES_CONTROLLER_ROLE_PATH, + KUBERNETES_CONTROLLER_ROLE_BINDING_PATH, + KUBERNETES_CONTROLLER_DEPLOYMENT_PATH, + KUBERNETES_CONTROLLER_SERVICE_PATH, + ], + )?; + ensure_kubernetes_router_allows_site_controller_control(artifact_root) +} + +fn build_embedded_kubernetes_controller_plan(plan: &SiteControllerPlan) -> SiteControllerPlan { + let mut embedded = plan.clone(); + embedded.listen_addr = SocketAddr::from(([0, 0, 0, 0], SITE_CONTROLLER_PORT)); + embedded.authority_url = + format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{SITE_CONTROLLER_PORT}"); + embedded.local_router_control = Some(format!( + "{KUBERNETES_ROUTER_NAME}:{KUBERNETES_ROUTER_CONTROL_PORT}" + )); + embedded.run_root = format!("{KUBERNETES_CONTROLLER_SITE_ROOT}/run"); + embedded.state_root = format!("{KUBERNETES_CONTROLLER_SITE_ROOT}/state-root"); + embedded.site_state_root = KUBERNETES_CONTROLLER_STATE_ROOT.to_string(); + embedded.artifact_dir = KUBERNETES_CONTROLLER_ARTIFACT_ROOT.to_string(); + embedded.state_path = KUBERNETES_CONTROLLER_STATE_PATH.to_string(); + embedded.context = None; + embedded.storage_root = None; + embedded.runtime_root = None; + embedded.compose_project = None; + embedded.kubernetes_namespace = plan.kubernetes_namespace.clone(); + embedded.router_mesh_port = Some(KUBERNETES_ROUTER_MESH_PORT); + embedded +} + +fn build_kubernetes_controller_seed_configmap( + artifact_root: &Path, + plan: &SiteControllerPlan, + embedded_plan: &SiteControllerPlan, +) -> Result { + let state_json = fs::read_to_string(&plan.state_path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", plan.state_path))?; + let desired_links = super::desired_links_path(Path::new(&plan.site_state_root)); + let desired_links_json = fs::read_to_string(&desired_links) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", desired_links.display()))?; + let embedded_plan_json = serde_json::to_string_pretty(embedded_plan) + .into_diagnostic() + .wrap_err("failed to serialize embedded kubernetes site controller plan")?; + Ok(json!({ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": format!("{SITE_CONTROLLER_SERVICE_NAME}-seed"), + "labels": kubernetes_controller_labels(), + }, + "data": { + "site-controller-plan.json": embedded_plan_json, + "site-controller-state.json": state_json, + "desired-links.json": desired_links_json, + "artifact.tar.b64": tar_directory_base64(artifact_root)?, + } + })) +} + +fn tar_directory_base64(root: &Path) -> Result { + let mut tar = Builder::new(Vec::new()); + tar.append_dir_all(".", root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to archive {}", root.display()))?; + let bytes = tar + .into_inner() + .into_diagnostic() + .wrap_err_with(|| format!("failed to finalize archive {}", root.display()))?; + Ok(base64::engine::general_purpose::STANDARD.encode(bytes)) +} + +fn add_kubernetes_resource_paths(artifact_root: &Path, paths: &[&str]) -> Result<()> { + let kustomization_path = artifact_root.join("kustomization.yaml"); + let raw = fs::read_to_string(&kustomization_path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", kustomization_path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid {}", kustomization_path.display()))?; + let resources = document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("resources"))) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!( + "kustomization {} is missing a resources sequence", + kustomization_path.display() + ) + })?; + for path in paths { + if resources + .iter() + .any(|value| value.as_str().is_some_and(|existing| existing == *path)) + { + continue; + } + resources.push(yaml_string(path)); + } + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", kustomization_path.display()))?; + fs::write(&kustomization_path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", kustomization_path.display())) +} + +fn ensure_kubernetes_router_allows_site_controller_control(artifact_root: &Path) -> Result<()> { + let path = artifact_root.join(KUBERNETES_ROUTER_NETPOL_PATH); + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid {}", path.display()))?; + let ingress = document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("spec"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .and_then(|spec| spec.get_mut(yaml_string("ingress"))) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!( + "router network policy {} is missing spec.ingress", + path.display() + ) + })?; + let controller_selector = kubernetes_controller_selector(); + let exists = ingress.iter().any(|rule| { + rule.as_mapping() + .and_then(|mapping| mapping.get(yaml_string("ports"))) + .and_then(serde_yaml::Value::as_sequence) + .is_some_and(|ports| { + ports.iter().any(|port| { + port.as_mapping() + .and_then(|entry| entry.get(yaml_string("port"))) + .and_then(serde_yaml::Value::as_i64) + == Some(i64::from(KUBERNETES_ROUTER_CONTROL_PORT)) + }) + }) + && rule + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("from"))) + .and_then(serde_yaml::Value::as_sequence) + .is_some_and(|from| { + from.iter().any(|peer| { + peer.as_mapping() + .and_then(|mapping| mapping.get(yaml_string("podSelector"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|selector_value| { + selector_value.get(yaml_string("matchLabels")) + }) + .and_then(serde_yaml::Value::as_mapping) + .is_some_and(|labels| { + labels.get(yaml_string("amber.io/component")) + == Some(&yaml_string( + controller_selector + .get("amber.io/component") + .expect("selector must contain component"), + )) + }) + }) + }) + }); + if !exists { + ingress.push( + serde_yaml::to_value(json!({ + "from": [{ + "podSelector": { + "matchLabels": controller_selector, + } + }], + "ports": [{ + "protocol": "TCP", + "port": KUBERNETES_ROUTER_CONTROL_PORT, + }] + })) + .into_diagnostic() + .wrap_err("failed to serialize site controller router ingress rule")?, + ); + } + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(&path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +fn kubernetes_controller_labels() -> BTreeMap { + BTreeMap::from([ + ( + "app.kubernetes.io/managed-by".to_string(), + "amber".to_string(), + ), + ( + "amber.io/component".to_string(), + SITE_CONTROLLER_SERVICE_NAME.to_string(), + ), + ("amber.io/type".to_string(), "site-controller".to_string()), + ]) +} + +fn kubernetes_controller_selector() -> BTreeMap { + BTreeMap::from([( + "amber.io/component".to_string(), + SITE_CONTROLLER_SERVICE_NAME.to_string(), + )]) +} + +fn write_yaml_artifact(path: PathBuf, value: &serde_json::Value) -> Result<()> { + let parent = path.parent().ok_or_else(|| { + miette::miette!("artifact path {} has no parent directory", path.display()) + })?; + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + let rendered = serde_yaml::to_string(value) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + let mut file = fs::File::create(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", path.display()))?; + file.write_all(rendered.as_bytes()) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} diff --git a/runtime/site-controller/src/default_runtime/site_artifacts.rs b/runtime/site-controller/src/default_runtime/site_artifacts.rs new file mode 100644 index 00000000..a9017d09 --- /dev/null +++ b/runtime/site-controller/src/default_runtime/site_artifacts.rs @@ -0,0 +1,2435 @@ +use super::*; + +fn compose_component_mesh_peer_addr( + artifact_root: &Path, + component_id: &str, + output: &MeshProvisionOutput, + mesh_port: u16, +) -> Result { + let MeshProvisionOutput::Filesystem { dir } = output else { + return Err(miette::miette!( + "compose artifact {} component {} does not use filesystem mesh output", + artifact_root.display(), + component_id + )); + }; + let service_name = Path::new(dir) + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| { + miette::miette!( + "compose artifact {} component {} has invalid mesh output dir {}", + artifact_root.display(), + component_id, + dir + ) + })?; + Ok(format!("{service_name}:{mesh_port}")) +} + +fn kubernetes_component_mesh_peer_addr( + artifact_root: &Path, + component_id: &str, + output: &MeshProvisionOutput, + mesh_port: u16, +) -> Result { + let MeshProvisionOutput::KubernetesSecret { name, .. } = output else { + return Err(miette::miette!( + "kubernetes artifact {} component {} does not use a kubernetes secret mesh output", + artifact_root.display(), + component_id + )); + }; + let service_name = name.strip_suffix("-mesh").ok_or_else(|| { + miette::miette!( + "kubernetes artifact {} component {} uses invalid mesh secret name {}", + artifact_root.display(), + component_id, + name + ) + })?; + Ok(format!("{service_name}:{mesh_port}")) +} + +fn build_dynamic_compose_route_overlay_payload( + artifact_root: &Path, + assigned_components: &[String], + component_mesh_dirs: &BTreeMap, + proxy_exports: &BTreeMap, + routed_inputs: &[DynamicInputRouteRecord], + existing_site_peer_identities: &BTreeMap, +) -> Result { + let plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; + let assigned = assigned_components + .iter() + .map(String::as_str) + .collect::>(); + let kept_component_ids = plan + .targets + .iter() + .filter(|target| { + matches!(target.kind, MeshProvisionTargetKind::Component) + && assigned.contains(target.config.identity.id.as_str()) + }) + .map(|target| target.config.identity.id.clone()) + .collect::>(); + let mut router_target = plan + .targets + .iter() + .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) + .cloned() + .ok_or_else(|| { + miette::miette!( + "compose child artifact {} is missing a router mesh target", + artifact_root.join("compose.yaml").display() + ) + })?; + filter_dynamic_router_target(&mut router_target, &kept_component_ids); + let component_peer_addrs = plan + .targets + .iter() + .filter(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) + .map(|target| { + Ok(( + target.config.identity.id.clone(), + compose_component_mesh_peer_addr( + artifact_root, + &target.config.identity.id, + &target.output, + target.config.mesh_listen.port(), + )?, + )) + }) + .collect::>>()?; + for route in &mut router_target.config.inbound { + if let InboundTarget::MeshForward { + peer_id, peer_addr, .. + } = &mut route.target + && let Some(resolved) = component_peer_addrs.get(peer_id) + { + *peer_addr = resolved.clone(); + } + } + + let component_mesh_scopes = plan + .targets + .iter() + .filter(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) + .map(|target| { + ( + target.config.identity.id.clone(), + target.config.identity.mesh_scope.clone(), + ) + }) + .collect::>(); + let mut peer_identities = existing_site_peer_identities.clone(); + for (component, relative_dir) in component_mesh_dirs { + let identity: MeshIdentitySecret = read_json( + &artifact_root + .join(relative_dir) + .join(MESH_IDENTITY_FILENAME), + "mesh identity", + )?; + peer_identities.insert( + component.clone(), + MeshIdentityPublic { + id: identity.id.clone(), + public_key: identity.public_key().into_diagnostic()?, + mesh_scope: component_mesh_scopes.get(component).cloned().flatten(), + }, + ); + } + let peers = router_target + .config + .peers + .iter() + .map(|peer| { + let identity = peer_identities.get(&peer.id).ok_or_else(|| { + miette::miette!( + "compose child router overlay peer {} is missing a live mesh identity", + peer.id + ) + })?; + Ok(MeshPeer { + id: identity.id.clone(), + public_key: identity.public_key, + }) + }) + .collect::>>()?; + + let allowed_issuers = overlay_issuer_sets(routed_inputs)?; + let mut payload = StoredRouteOverlayPayload { + peers, + inbound_routes: router_target.config.inbound, + }; + augment_route_overlay_payload( + &mut payload, + proxy_exports, + routed_inputs, + &component_peer_addrs, + &peer_identities, + Some(&allowed_issuers), + false, + )?; + Ok(payload) +} + +fn rewrite_compose_mesh_bind_mounts( + artifact_root: &Path, + mesh_dirs: &BTreeMap, +) -> Result<()> { + let compose_path = artifact_root.join("compose.yaml"); + let mut document = read_compose_document(&compose_path)?; + let services = compose_services_mut(&mut document, &compose_path)?; + for (service_name, relative_dir) in mesh_dirs { + let service = services.get_mut(yaml_string(service_name)).ok_or_else(|| { + miette::miette!( + "compose file {} is missing child sidecar service {}", + compose_path.display(), + service_name + ) + })?; + let Some(service_mapping) = service.as_mapping_mut() else { + return Err(miette::miette!( + "compose service {} is not a mapping", + service_name + )); + }; + let volumes_key = yaml_string("volumes"); + let volumes = service_mapping + .get_mut(&volumes_key) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!( + "compose child sidecar {} is missing a volumes list", + service_name + ) + })?; + let expected_prefix = format!("{service_name}-mesh:/amber/mesh"); + let replacement = serde_yaml::Value::String(format!("./{relative_dir}:/amber/mesh:ro")); + let mut replaced = false; + for volume in volumes.iter_mut() { + if volume + .as_str() + .is_some_and(|value| value.starts_with(&expected_prefix)) + { + *volume = replacement.clone(); + replaced = true; + } + } + if !replaced { + volumes.push(replacement); + } + } + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", compose_path.display()))?; + fs::write(&compose_path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", compose_path.display())) +} + +pub(super) fn load_dynamic_compose_child_metadata( + artifact_root: &Path, +) -> Result { + read_json( + &dynamic_compose_child_metadata_path(artifact_root), + "dynamic compose child metadata", + ) +} + +fn load_running_site_router_identity( + plan: &SiteControllerRuntimePlan, +) -> Result { + Ok(load_live_site_router_mesh_config(plan)?.identity) +} + +fn filesystem_component_peer_identities_for_artifact( + artifact_root: &Path, + mesh_plan: &MeshProvisionPlan, + artifact_kind: &str, + compose_project: Option<&str>, +) -> Result> { + let mut peers = BTreeMap::new(); + for target in &mesh_plan.targets { + if !matches!(target.kind, MeshProvisionTargetKind::Component) { + continue; + } + let MeshProvisionOutput::Filesystem { dir } = &target.output else { + return Err(miette::miette!( + "{artifact_kind} artifact {} has non-filesystem mesh output for component {}", + artifact_root.display(), + target.config.identity.id + )); + }; + let config: MeshConfigPublic = if Path::new(dir).is_absolute() { + let compose_project = compose_project.ok_or_else(|| { + miette::miette!( + "{artifact_kind} artifact {} uses absolute mesh output {} without a compose \ + project", + artifact_root.display(), + dir + ) + })?; + let service_name = Path::new(dir) + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| { + miette::miette!( + "{artifact_kind} artifact {} has invalid absolute mesh output {}", + artifact_root.display(), + dir + ) + })?; + read_compose_volume_mesh_config(compose_project, service_name)? + } else { + read_json( + &artifact_root.join(dir).join(MESH_CONFIG_FILENAME), + "mesh config", + )? + }; + peers.insert(config.identity.id.clone(), config.identity); + } + Ok(peers) +} + +fn read_compose_volume_mesh_config( + compose_project: &str, + service_name: &str, +) -> Result { + let volume_name = format!("{compose_project}_{service_name}-mesh"); + let output = Command::new("docker") + .arg("run") + .arg("--rm") + .arg("-v") + .arg(format!("{volume_name}:/amber/mesh:ro")) + .arg("busybox:1.36.1") + .arg("cat") + .arg(format!("/amber/mesh/{MESH_CONFIG_FILENAME}")) + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to read compose mesh config from docker volume {volume_name}") + })?; + if !output.status.success() { + return Err(miette::miette!( + "failed to read compose mesh config from docker volume \ + {volume_name}\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + )); + } + serde_json::from_slice(&output.stdout) + .into_diagnostic() + .wrap_err_with(|| format!("docker volume {volume_name} returned invalid mesh config json")) +} + +fn compose_peer_identities_for_artifact( + artifact_root: &Path, + compose_project: Option<&str>, +) -> Result> { + let mesh_plan = if artifact_root.join("mesh-provision-plan.json").is_file() { + read_json( + &artifact_root.join("mesh-provision-plan.json"), + "mesh provision plan", + )? + } else { + read_embedded_compose_mesh_provision_plan(artifact_root)? + }; + filesystem_component_peer_identities_for_artifact( + artifact_root, + &mesh_plan, + "compose", + compose_project, + ) +} + +pub(super) fn local_compose_peer_identities( + plan: &SiteControllerRuntimePlan, + published_children: &[SiteControllerRuntimeChildRecord], +) -> Result> { + let mut peers = BTreeMap::new(); + let router = load_running_site_router_identity(plan)?; + peers.insert(router.id.clone(), router); + peers.extend(compose_peer_identities_for_artifact( + Path::new(&plan.artifact_dir), + plan.compose_project.as_deref(), + )?); + for child in published_children { + peers.extend(compose_peer_identities_for_artifact( + Path::new(&child.artifact_root), + plan.compose_project.as_deref(), + )?); + } + Ok(peers) +} + +fn kubernetes_peer_identities_for_artifact( + plan: &SiteControllerRuntimePlan, + artifact_root: &Path, +) -> Result> { + let mesh_plan = read_embedded_kubernetes_mesh_provision_plan(artifact_root)?; + let mut peers = BTreeMap::new(); + for target in &mesh_plan.targets { + if !matches!(target.kind, MeshProvisionTargetKind::Component) { + continue; + } + let MeshProvisionOutput::KubernetesSecret { name, namespace } = &target.output else { + return Err(miette::miette!( + "kubernetes artifact {} has non-secret mesh output for component {}", + artifact_root.display(), + target.config.identity.id + )); + }; + let config = load_kubernetes_mesh_config_public(plan, name, namespace.as_deref())?; + peers.insert(config.identity.id.clone(), config.identity); + } + Ok(peers) +} + +pub(super) fn local_kubernetes_peer_identities( + plan: &SiteControllerRuntimePlan, + published_children: &[SiteControllerRuntimeChildRecord], +) -> Result> { + let mut peers = BTreeMap::new(); + let router = load_running_site_router_identity(plan)?; + peers.insert(router.id.clone(), router); + peers.extend(kubernetes_peer_identities_for_artifact( + plan, + Path::new(&plan.artifact_dir), + )?); + for child in published_children { + peers.extend(kubernetes_peer_identities_for_artifact( + plan, + Path::new(&child.artifact_root), + )?); + } + Ok(peers) +} + +pub(crate) fn collect_live_component_runtime_metadata( + plan: &SiteControllerRuntimePlan, +) -> Result> { + let state: SiteControllerRuntimeState = read_json( + &site_controller_runtime_state_path(Path::new(&plan.site_state_root)), + "site controller runtime state", + )?; + let published_children = state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>(); + let mut components = match plan.kind { + SiteKind::Direct => collect_direct_artifact_runtime_metadata( + Path::new(&plan.artifact_dir), + Path::new(plan.runtime_root.as_deref().ok_or_else(|| { + miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) + })?), + )?, + SiteKind::Vm => collect_vm_artifact_runtime_metadata( + Path::new(&plan.artifact_dir), + Path::new(plan.runtime_root.as_deref().ok_or_else(|| { + miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) + })?), + )?, + SiteKind::Compose => collect_compose_artifact_runtime_metadata( + Path::new(&plan.artifact_dir), + plan.compose_project.as_deref(), + )?, + SiteKind::Kubernetes => { + collect_kubernetes_artifact_runtime_metadata(plan, Path::new(&plan.artifact_dir))? + } + }; + for child in &published_children { + let child_components = match plan.kind { + SiteKind::Direct => collect_direct_artifact_runtime_metadata( + Path::new(&child.artifact_root), + &site_controller_runtime_child_runtime_root(plan, child.child_id), + )?, + SiteKind::Vm => collect_vm_artifact_runtime_metadata( + Path::new(&child.artifact_root), + &site_controller_runtime_child_runtime_root(plan, child.child_id), + )?, + SiteKind::Compose => collect_compose_artifact_runtime_metadata( + Path::new(&child.artifact_root), + plan.compose_project.as_deref(), + )?, + SiteKind::Kubernetes => { + collect_kubernetes_artifact_runtime_metadata(plan, Path::new(&child.artifact_root))? + } + }; + components.extend(child_components); + } + Ok(components) +} + +pub(crate) fn load_live_site_router_mesh_config( + plan: &SiteControllerRuntimePlan, +) -> Result { + let artifact_root = Path::new(&plan.artifact_dir); + match plan.kind { + SiteKind::Direct => { + let runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { + miette::miette!("direct site `{}` is missing its runtime root", plan.site_id) + })?); + let direct_plan: DirectPlan = + read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; + let router = direct_plan.router.ok_or_else(|| { + miette::miette!("direct site `{}` is missing its router plan", plan.site_id) + })?; + read_json(&runtime_root.join(&router.mesh_config_path), "mesh config") + } + SiteKind::Vm => { + let runtime_root = Path::new(plan.runtime_root.as_deref().ok_or_else(|| { + miette::miette!("vm site `{}` is missing its runtime root", plan.site_id) + })?); + let vm_plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; + let router = vm_plan.router.ok_or_else(|| { + miette::miette!("vm site `{}` is missing its router plan", plan.site_id) + })?; + read_json(&runtime_root.join(&router.mesh_config_path), "mesh config") + } + SiteKind::Compose => { + let mesh_plan = if artifact_root.join("mesh-provision-plan.json").is_file() { + read_json( + &artifact_root.join("mesh-provision-plan.json"), + "mesh provision plan", + )? + } else { + read_embedded_compose_mesh_provision_plan(artifact_root)? + }; + let target = mesh_plan + .targets + .iter() + .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) + .ok_or_else(|| { + miette::miette!( + "compose site `{}` is missing a router mesh target", + plan.site_id + ) + })?; + let MeshProvisionOutput::Filesystem { dir } = &target.output else { + return Err(miette::miette!( + "compose site `{}` has non-filesystem mesh output for router {}", + plan.site_id, + target.config.identity.id + )); + }; + if Path::new(dir).is_absolute() { + let compose_project = plan.compose_project.as_deref().ok_or_else(|| { + miette::miette!( + "compose site `{}` is missing its compose project", + plan.site_id + ) + })?; + let service_name = Path::new(dir) + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| { + miette::miette!( + "compose site `{}` has invalid router mesh output {}", + plan.site_id, + dir + ) + })?; + read_compose_volume_mesh_config(compose_project, service_name) + } else { + read_json( + &artifact_root.join(dir).join(MESH_CONFIG_FILENAME), + "mesh config", + ) + } + } + SiteKind::Kubernetes => { + let mesh_plan = read_kubernetes_runtime_mesh_provision_plan(artifact_root)?; + let target = mesh_plan + .targets + .iter() + .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) + .ok_or_else(|| { + miette::miette!( + "kubernetes site `{}` is missing a router mesh target", + plan.site_id + ) + })?; + let MeshProvisionOutput::KubernetesSecret { name, namespace } = &target.output else { + return Err(miette::miette!( + "kubernetes site `{}` has non-secret mesh output for router {}", + plan.site_id, + target.config.identity.id + )); + }; + load_kubernetes_mesh_config_public(plan, name, namespace.as_deref()) + } + } +} + +fn collect_direct_artifact_runtime_metadata( + artifact_root: &Path, + runtime_root: &Path, +) -> Result> { + let state: DirectRuntimeState = read_json( + &direct_runtime_state_path(artifact_root), + "direct runtime state", + )?; + let plan: DirectPlan = read_json(&artifact_root.join("direct-plan.json"), "direct plan")?; + let mut components = BTreeMap::new(); + for component in &plan.components { + let mesh_config: MeshConfigPublic = read_json( + &runtime_root.join(&component.sidecar.mesh_config_path), + "mesh config", + )?; + let mesh_port = state + .component_mesh_port_by_id + .get(&component.id) + .copied() + .ok_or_else(|| { + miette::miette!( + "direct runtime state is missing mesh port for component {}", + component.moniker + ) + })?; + components.insert( + component.moniker.clone(), + LiveComponentRuntimeMetadata { + moniker: component.moniker.clone(), + host_mesh_addr: format!("127.0.0.1:{mesh_port}"), + control_endpoint: Some(ControlEndpoint::Unix( + direct_component_control_socket_path( + runtime_root.join(&component.program.work_dir), + component.id, + ), + )), + mesh_config, + }, + ); + } + Ok(components) +} + +fn collect_vm_artifact_runtime_metadata( + artifact_root: &Path, + runtime_root: &Path, +) -> Result> { + let state = load_vm_runtime_state_for_artifact(artifact_root, runtime_root)?; + let plan: VmPlan = read_json(&artifact_root.join("vm-plan.json"), "vm plan")?; + let mut components = BTreeMap::new(); + for component in &plan.components { + let mesh_config: MeshConfigPublic = read_json( + &runtime_root.join(&component.mesh_config_path), + "mesh config", + )?; + let mesh_port = state + .component_mesh_port_by_id + .get(&component.id) + .copied() + .ok_or_else(|| { + miette::miette!( + "vm runtime state is missing mesh port for component {}", + component.moniker + ) + })?; + components.insert( + component.moniker.clone(), + LiveComponentRuntimeMetadata { + moniker: component.moniker.clone(), + host_mesh_addr: format!("127.0.0.1:{mesh_port}"), + control_endpoint: Some(ControlEndpoint::Unix(vm_component_control_socket_path( + runtime_root + .join("work") + .join("sidecars") + .join(&component.log_name), + component.id, + ))), + mesh_config, + }, + ); + } + Ok(components) +} + +fn direct_component_control_socket_path(work_dir: PathBuf, component_id: usize) -> PathBuf { + amber_mesh::stable_temp_socket_path( + "amber-direct-control", + &format!("sidecar-{component_id}"), + &work_dir, + ) +} + +fn vm_component_control_socket_path(work_dir: PathBuf, component_id: usize) -> PathBuf { + amber_mesh::stable_temp_socket_path( + "amber-vm-control", + &format!("sidecar-{component_id}"), + &work_dir, + ) +} + +fn collect_compose_artifact_runtime_metadata( + artifact_root: &Path, + compose_project: Option<&str>, +) -> Result> { + let mesh_plan = if artifact_root.join("mesh-provision-plan.json").is_file() { + read_json( + &artifact_root.join("mesh-provision-plan.json"), + "mesh provision plan", + )? + } else { + read_embedded_compose_mesh_provision_plan(artifact_root)? + }; + let mut components = BTreeMap::new(); + for target in &mesh_plan.targets { + if !matches!(target.kind, MeshProvisionTargetKind::Component) { + continue; + } + let MeshProvisionOutput::Filesystem { dir } = &target.output else { + return Err(miette::miette!( + "compose artifact {} has non-filesystem mesh output for component {}", + artifact_root.display(), + target.config.identity.id + )); + }; + let service_name = Path::new(dir) + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| { + miette::miette!( + "compose artifact {} has invalid mesh output {} for component {}", + artifact_root.display(), + dir, + target.config.identity.id + ) + })?; + let mesh_config = if Path::new(dir).is_absolute() { + let compose_project = compose_project.ok_or_else(|| { + miette::miette!( + "compose artifact {} requires a compose project to resolve mesh output {}", + artifact_root.display(), + dir + ) + })?; + read_compose_volume_mesh_config(compose_project, service_name)? + } else { + read_json( + &artifact_root.join(dir).join(MESH_CONFIG_FILENAME), + "mesh config", + )? + }; + components.insert( + target.config.identity.id.clone(), + LiveComponentRuntimeMetadata { + moniker: target.config.identity.id.clone(), + host_mesh_addr: compose_component_mesh_peer_addr( + artifact_root, + &target.config.identity.id, + &target.output, + mesh_config.mesh_listen.port(), + )?, + control_endpoint: compose_project.map(|compose_project| { + ControlEndpoint::VolumeSocket { + volume: compose_component_control_socket_volume_name( + compose_project, + service_name, + ), + socket_path: COMPONENT_CONTROL_SOCKET_PATH_IN_CONTAINER.to_string(), + } + }), + mesh_config, + }, + ); + } + Ok(components) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn direct_runtime_metadata_uses_bounded_sidecar_control_socket_paths() { + let work_dir = PathBuf::from( + "/Users/example/Developer/amber/target/cli-test-outputs/\ + mixed-run-doc-example-detach-very-long/state/runs/run-123/state/direct_local/runtime/\ + work/components/c1-admin", + ); + let socket = direct_component_control_socket_path(work_dir.clone(), 7); + let rendered = socket.display().to_string(); + + assert_eq!( + socket, + amber_mesh::stable_temp_socket_path("amber-direct-control", "sidecar-7", &work_dir) + ); + assert!( + rendered.len() < 104, + "direct runtime metadata should use the hashed short socket path: {rendered}", + ); + } + + #[test] + fn vm_runtime_metadata_uses_bounded_sidecar_control_socket_paths() { + let work_dir = PathBuf::from( + "/Users/example/Developer/amber/target/cli-test-outputs/\ + linux-vm-framework_component-very-long/state/runs/run-123/state/vm_local/runtime/\ + work/sidecars/c1-worker-sidecar", + ); + let socket = vm_component_control_socket_path(work_dir.clone(), 11); + let rendered = socket.display().to_string(); + + assert_eq!( + socket, + amber_mesh::stable_temp_socket_path("amber-vm-control", "sidecar-11", &work_dir) + ); + assert!( + rendered.len() < 104, + "vm runtime metadata should use the hashed short socket path: {rendered}", + ); + } +} + +fn collect_kubernetes_artifact_runtime_metadata( + plan: &SiteControllerRuntimePlan, + artifact_root: &Path, +) -> Result> { + let mesh_plan = read_kubernetes_runtime_mesh_provision_plan(artifact_root)?; + let mut components = BTreeMap::new(); + for target in &mesh_plan.targets { + if !matches!(target.kind, MeshProvisionTargetKind::Component) { + continue; + } + let MeshProvisionOutput::KubernetesSecret { name, namespace } = &target.output else { + return Err(miette::miette!( + "kubernetes artifact {} has non-secret mesh output for component {}", + artifact_root.display(), + target.config.identity.id + )); + }; + name.strip_suffix("-mesh").ok_or_else(|| { + miette::miette!( + "kubernetes artifact {} component {} uses invalid mesh secret name {}", + artifact_root.display(), + target.config.identity.id, + name + ) + })?; + let mesh_config = load_kubernetes_mesh_config_public(plan, name, namespace.as_deref())?; + components.insert( + target.config.identity.id.clone(), + LiveComponentRuntimeMetadata { + moniker: target.config.identity.id.clone(), + host_mesh_addr: kubernetes_component_mesh_peer_addr( + artifact_root, + &target.config.identity.id, + &target.output, + mesh_config.mesh_listen.port(), + )?, + control_endpoint: None, + mesh_config, + }, + ); + } + Ok(components) +} + +fn compose_component_control_socket_volume_name( + compose_project: &str, + service_name: &str, +) -> String { + format!("{compose_project}_{service_name}-control") +} + +pub(super) fn prepare_dynamic_compose_child_artifact( + plan: &SiteControllerRuntimePlan, + runtime_spec: &LocalChildRuntimeSpec, + artifact_root: &Path, + published_children: &[SiteControllerRuntimeChildRecord], + existing_site_peer_identities: &BTreeMap, + live_components: &BTreeMap, +) -> Result<()> { + project_dynamic_child_mesh_scope(artifact_root, Some(&plan.mesh_scope))?; + let compose_path = artifact_root.join("compose.yaml"); + let mut desired_document = read_compose_document(&compose_path)?; + let root_services = + compose_dynamic_root_service_names(artifact_root, &runtime_spec.assigned_components)?; + let service_closure = + compose_service_closure(&desired_document, &compose_path, &root_services)?; + let live_services = compose_live_service_names(plan, published_children)?; + let mut kept_services = service_closure + .difference(&live_services) + .cloned() + .collect::>(); + kept_services.remove(COMPOSE_PROVISIONER_SERVICE_NAME); + if kept_services.is_empty() { + return Err(miette::miette!( + "compose child artifact {} does not retain any child-owned services after filtering", + compose_path.display() + )); + } + + let services = compose_services_mut(&mut desired_document, &compose_path)?; + services.retain(|name, _| { + name.as_str() + .is_some_and(|service_name| kept_services.contains(service_name)) + }); + for service in services.values_mut() { + retain_compose_service_dependencies(service, &kept_services)?; + } + let rendered = serde_yaml::to_string(&desired_document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", compose_path.display()))?; + fs::write(&compose_path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", compose_path.display()))?; + + let DynamicComposeMeshPlan { + mesh_plan, + mesh_dirs, + component_mesh_dirs, + } = build_dynamic_compose_mesh_plan(artifact_root, &runtime_spec.assigned_components)?; + let mut mesh_plan = mesh_plan; + let router_mesh_port = router_mesh_port_from_plan( + &read_embedded_compose_mesh_provision_plan(artifact_root)?, + "compose", + )?; + ensure_dynamic_proxy_export_component_routes( + &mut mesh_plan, + &runtime_spec.proxy_exports, + &plan.router_identity_id, + )?; + rewrite_dynamic_direct_inputs(&mut mesh_plan, &runtime_spec.direct_inputs, live_components)?; + rewrite_dynamic_routed_inputs( + &mut mesh_plan, + &runtime_spec.routed_inputs, + SiteKind::Compose, + &plan.router_identity_id, + Some(router_mesh_port), + )?; + let existing_peer_identities = + required_existing_mesh_peer_identities(&mesh_plan, existing_site_peer_identities)?; + mesh_plan.existing_peer_identities = existing_peer_identities.values().cloned().collect(); + write_json(&artifact_root.join("mesh-provision-plan.json"), &mesh_plan)?; + provision_mesh_filesystem_with_peer_identities( + &mesh_plan, + artifact_root, + &existing_peer_identities, + )?; + for relative_dir in mesh_dirs.values() { + project_existing_peer_identities_into_mesh_config( + &artifact_root.join(relative_dir).join(MESH_CONFIG_FILENAME), + &existing_peer_identities, + )?; + } + rewrite_compose_mesh_bind_mounts(artifact_root, &mesh_dirs)?; + let overlay_payload = build_dynamic_compose_route_overlay_payload( + artifact_root, + &runtime_spec.assigned_components, + &component_mesh_dirs, + &runtime_spec.proxy_exports, + &runtime_spec.routed_inputs, + existing_site_peer_identities, + )?; + write_json(&dynamic_route_overlay_path(artifact_root), &overlay_payload)?; + write_embedded_compose_mesh_provision_plan(artifact_root, &mesh_plan)?; + + write_json( + &dynamic_compose_child_metadata_path(artifact_root), + &DynamicComposeChildMetadata { + schema: DYNAMIC_COMPOSE_CHILD_SCHEMA.to_string(), + version: DYNAMIC_COMPOSE_CHILD_VERSION, + services: kept_services.iter().cloned().collect(), + readiness_services: root_services + .into_iter() + .filter(|service| kept_services.contains(service)) + .collect(), + }, + ) +} + +pub(super) fn read_compose_document(path: &Path) -> Result { + let raw = fs::read_to_string(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid compose file {}", path.display())) +} + +pub(super) fn read_embedded_compose_mesh_provision_plan( + artifact_root: &Path, +) -> Result { + let path = artifact_root.join("compose.yaml"); + let document = read_compose_document(&path)?; + let Some(root) = document.as_mapping() else { + return Err(miette::miette!( + "compose file {} is not a YAML mapping", + path.display() + )); + }; + let configs_key = serde_yaml::Value::String("configs".to_string()); + let config_name = serde_yaml::Value::String("amber-mesh-provision-plan".to_string()); + let content_key = serde_yaml::Value::String("content".to_string()); + let content = root + .get(&configs_key) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|configs| configs.get(&config_name)) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|config| config.get(&content_key)) + .and_then(serde_yaml::Value::as_str) + .ok_or_else(|| { + miette::miette!( + "compose file {} is missing configs.amber-mesh-provision-plan.content", + path.display() + ) + })?; + serde_json::from_str(content).map_err(|err| { + miette::miette!( + "compose file {} has invalid embedded mesh provision plan: {err}", + path.display() + ) + }) +} + +fn write_embedded_compose_mesh_provision_plan( + artifact_root: &Path, + plan: &MeshProvisionPlan, +) -> Result<()> { + let path = artifact_root.join("compose.yaml"); + let mut document = read_compose_document(&path)?; + let plan_json = serde_json::to_string_pretty(plan) + .into_diagnostic() + .wrap_err("failed to serialize compose mesh provision plan")?; + document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("configs"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .and_then(|configs| configs.get_mut(yaml_string("amber-mesh-provision-plan"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or_else(|| { + miette::miette!( + "compose file {} is missing configs.amber-mesh-provision-plan", + path.display() + ) + })? + .insert(yaml_string("content"), serde_yaml::Value::String(plan_json)); + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(&path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +fn read_embedded_kubernetes_mesh_provision_plan(artifact_root: &Path) -> Result { + let path = artifact_root + .join("01-configmaps") + .join("amber-mesh-provision.yaml"); + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let document: serde_yaml::Value = + serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "invalid kubernetes mesh provision configmap {}", + path.display() + ) + })?; + let mesh_plan = document + .as_mapping() + .and_then(|root| root.get(yaml_string("data"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|data| data.get(yaml_string("mesh-plan.json"))) + .and_then(serde_yaml::Value::as_str) + .ok_or_else(|| { + miette::miette!( + "kubernetes mesh provision configmap {} is missing data.mesh-plan.json", + path.display() + ) + })?; + serde_json::from_str(mesh_plan).map_err(|err| { + miette::miette!( + "kubernetes mesh provision configmap {} has invalid mesh plan: {err}", + path.display() + ) + }) +} + +fn read_kubernetes_runtime_mesh_provision_plan(artifact_root: &Path) -> Result { + let path = artifact_root.join("mesh-provision-plan.json"); + if path.is_file() { + return read_json(&path, "mesh provision plan"); + } + read_embedded_kubernetes_mesh_provision_plan(artifact_root) +} + +fn write_embedded_kubernetes_mesh_provision_plan( + artifact_root: &Path, + plan: &MeshProvisionPlan, +) -> Result<()> { + let path = artifact_root.join(KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH); + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "invalid kubernetes mesh provision configmap {}", + path.display() + ) + })?; + let plan_json = serde_json::to_string_pretty(plan) + .into_diagnostic() + .wrap_err("failed to serialize kubernetes mesh provision plan")?; + document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("data"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or_else(|| { + miette::miette!( + "kubernetes mesh provision configmap {} is missing a data mapping", + path.display() + ) + })? + .insert( + yaml_string("mesh-plan.json"), + serde_yaml::Value::String(plan_json), + ); + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(&path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +#[derive(Clone, Debug)] +pub struct SiteControllerPeerRouterRoute { + pub site_id: String, + pub peer_router: MeshIdentityPublic, + pub peer_addr: String, + pub listen_addr: String, + pub listen_port: u16, +} + +pub fn inject_site_controller_peer_router_routes( + artifact_root: &Path, + local_site_id: &str, + allowed_issuers: &[String], + routes: &[SiteControllerPeerRouterRoute], +) -> Result<()> { + let mut write_embedded = None; + let mut plan = if artifact_root.join("mesh-provision-plan.json").is_file() { + read_json( + &artifact_root.join("mesh-provision-plan.json"), + "mesh provision plan", + )? + } else if artifact_root.join("compose.yaml").is_file() { + write_embedded = Some(EmbeddedMeshPlanKind::Compose); + read_embedded_compose_mesh_provision_plan(artifact_root)? + } else if artifact_root + .join(KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH) + .is_file() + { + write_embedded = Some(EmbeddedMeshPlanKind::Kubernetes); + read_embedded_kubernetes_mesh_provision_plan(artifact_root)? + } else { + return Err(miette::miette!( + "site artifact {} is missing a mesh provision plan", + artifact_root.display() + )); + }; + + inject_site_controller_peer_router_routes_into_plan( + &mut plan, + local_site_id, + allowed_issuers, + routes, + )?; + + match write_embedded { + None => write_json(&artifact_root.join("mesh-provision-plan.json"), &plan), + Some(EmbeddedMeshPlanKind::Compose) => { + write_embedded_compose_mesh_provision_plan(artifact_root, &plan) + } + Some(EmbeddedMeshPlanKind::Kubernetes) => { + write_embedded_kubernetes_mesh_provision_plan(artifact_root, &plan) + } + } +} + +pub fn set_compose_router_published_mesh_port(artifact_root: &Path, host_port: u16) -> Result<()> { + set_compose_router_published_port(artifact_root, "0.0.0.0", host_port, 24000, false) +} + +pub fn add_compose_router_published_route_ports( + artifact_root: &Path, + route_ports: &[u16], +) -> Result<()> { + for port in route_ports { + set_compose_router_published_port(artifact_root, "127.0.0.1", *port, *port, true)?; + } + Ok(()) +} + +fn set_compose_router_published_port( + artifact_root: &Path, + host_ip: &str, + host_port: u16, + container_port: u16, + append_if_missing: bool, +) -> Result<()> { + let path = artifact_root.join("compose.yaml"); + let mut document = read_compose_document(&path)?; + let router_ports = document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("services"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .and_then(|services| services.get_mut(yaml_string(COMPOSE_ROUTER_SERVICE_NAME))) + .and_then(serde_yaml::Value::as_mapping_mut) + .and_then(|router| router.get_mut(yaml_string("ports"))) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!( + "compose file {} is missing services.{COMPOSE_ROUTER_SERVICE_NAME}.ports", + path.display() + ) + })?; + let desired = format!("{host_ip}:{host_port}:{container_port}"); + let mut updated = false; + for port in &mut *router_ports { + let Some(raw) = port.as_str() else { + continue; + }; + if raw == desired { + updated = true; + continue; + } + if raw.ends_with(&format!("::{container_port}")) { + *port = serde_yaml::Value::String(desired.clone()); + updated = true; + } + } + if !updated && append_if_missing { + router_ports.push(serde_yaml::Value::String(desired)); + updated = true; + } + if !updated { + return Err(miette::miette!( + "compose file {} has no router publish to rewrite for container port {container_port}", + path.display(), + )); + } + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(&path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +enum EmbeddedMeshPlanKind { + Compose, + Kubernetes, +} + +fn inject_site_controller_peer_router_routes_into_plan( + plan: &mut MeshProvisionPlan, + local_site_id: &str, + allowed_issuers: &[String], + routes: &[SiteControllerPeerRouterRoute], +) -> Result<()> { + let router = plan + .targets + .iter_mut() + .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) + .ok_or_else(|| miette::miette!("mesh provision plan is missing a router target"))?; + let inbound_route_id = site_controller_internal_route_id(local_site_id); + router + .config + .inbound + .retain(|route| route.route_id != inbound_route_id); + router.config.inbound.push(InboundRoute { + route_id: inbound_route_id, + capability: SITE_CONTROLLER_INTERNAL_CAPABILITY.to_string(), + capability_kind: None, + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: InboundTarget::External { + url_env: amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV.to_string(), + optional: false, + }, + allowed_issuers: allowed_issuers.to_vec(), + }); + router + .config + .inbound + .sort_by(|left, right| left.route_id.cmp(&right.route_id)); + router + .config + .outbound + .retain(|route| !route.route_id.starts_with("site-controller:")); + let mut existing_peer_identities = plan + .existing_peer_identities + .iter() + .cloned() + .map(|identity| (identity.id.clone(), identity)) + .collect::>(); + for route in routes { + existing_peer_identities.insert(route.peer_router.id.clone(), route.peer_router.clone()); + if !router + .config + .peers + .iter() + .any(|peer| peer.id == route.peer_router.id) + { + router.config.peers.push(MeshPeerTemplate { + id: route.peer_router.id.clone(), + }); + } + router.config.outbound.push(OutboundRoute { + route_id: site_controller_internal_route_id(&route.site_id), + slot: route.site_id.clone(), + capability_kind: None, + capability_profile: None, + listen_port: route.listen_port, + listen_addr: Some(route.listen_addr.clone()), + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + peer_addr: route.peer_addr.clone(), + peer_id: route.peer_router.id.clone(), + capability: SITE_CONTROLLER_INTERNAL_CAPABILITY.to_string(), + }); + } + plan.existing_peer_identities = existing_peer_identities.into_values().collect(); + router + .config + .outbound + .sort_by(|left, right| left.route_id.cmp(&right.route_id)); + router + .config + .peers + .sort_by(|left, right| left.id.cmp(&right.id)); + Ok(()) +} + +pub fn set_site_artifact_mesh_identity_seed( + artifact_root: &Path, + identity_seed: &str, +) -> Result<()> { + let path = artifact_root.join("mesh-provision-plan.json"); + if path.is_file() { + let mut plan: MeshProvisionPlan = read_json(&path, "mesh provision plan")?; + if plan.identity_seed.as_deref() != Some(identity_seed) { + plan.identity_seed = Some(identity_seed.to_string()); + write_json(&path, &plan)?; + } + return Ok(()); + } + + let compose_path = artifact_root.join("compose.yaml"); + if compose_path.is_file() { + let mut plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; + if plan.identity_seed.as_deref() != Some(identity_seed) { + plan.identity_seed = Some(identity_seed.to_string()); + write_embedded_compose_mesh_provision_plan(artifact_root, &plan)?; + } + return Ok(()); + } + + let configmap_path = artifact_root.join(KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH); + if configmap_path.is_file() { + let mut plan = read_embedded_kubernetes_mesh_provision_plan(artifact_root)?; + if plan.identity_seed.as_deref() != Some(identity_seed) { + plan.identity_seed = Some(identity_seed.to_string()); + write_embedded_kubernetes_mesh_provision_plan(artifact_root, &plan)?; + } + return Ok(()); + } + + Err(miette::miette!( + "site artifact {} is missing a mesh provision plan", + artifact_root.display() + )) +} + +fn kubernetes_resource_name(document: &serde_yaml::Value) -> Option<&str> { + document + .as_mapping() + .and_then(|root| root.get(yaml_string("metadata"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|metadata| metadata.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) +} + +fn kubernetes_dynamic_apply_resource_kept_from_contents( + resource: &str, + raw: &str, + child_component_labels: &BTreeSet, +) -> Result { + if matches!( + resource, + KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH + | KUBERNETES_PROVISIONER_JOB_PATH + | KUBERNETES_PROVISIONER_ROLE_PATH + | KUBERNETES_PROVISIONER_ROLEBINDING_PATH + | KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH + ) || resource.starts_with("03-persistentvolumeclaims/") + { + return Ok(true); + } + + let document: serde_yaml::Value = serde_yaml::from_str(raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid kubernetes resource {resource}"))?; + Ok(document + .as_mapping() + .and_then(|root| root.get(yaml_string("metadata"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|metadata| metadata.get(yaml_string("labels"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|labels| labels.get(yaml_string("amber.io/component-id"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|component_id| child_component_labels.contains(component_id))) +} + +pub(crate) fn project_kubernetes_dynamic_child_artifact_files( + artifact_files: &BTreeMap, + component_ids: &[usize], +) -> Result> { + let child_component_labels = component_ids + .iter() + .map(|component_id| format!("c{component_id}")) + .collect::>(); + let kustomization_path = "kustomization.yaml"; + let raw = artifact_files.get(kustomization_path).ok_or_else(|| { + miette::miette!("dynamic kubernetes artifact snapshot is missing {kustomization_path}") + })?; + let mut document: serde_yaml::Value = serde_yaml::from_str(raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid kustomization {kustomization_path}"))?; + let root = document.as_mapping_mut().ok_or_else(|| { + miette::miette!("kustomization {kustomization_path} is not a YAML mapping") + })?; + let resources = root + .get_mut(yaml_string("resources")) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!("kustomization {kustomization_path} is missing a resources sequence") + })?; + let mut projected = artifact_files + .iter() + .filter(|(path, _)| !path.ends_with(".yaml") && path.as_str() != kustomization_path) + .map(|(path, contents)| (path.clone(), contents.clone())) + .collect::>(); + let mut kept_resources = Vec::new(); + let mut kept_resource_names = BTreeSet::new(); + for resource in resources + .iter() + .filter_map(serde_yaml::Value::as_str) + .map(str::to_owned) + { + let raw = artifact_files + .get(&resource) + .ok_or_else(|| miette::miette!("dynamic kubernetes artifact is missing {resource}"))?; + if !kubernetes_dynamic_apply_resource_kept_from_contents( + &resource, + raw, + &child_component_labels, + )? { + continue; + } + let document: serde_yaml::Value = serde_yaml::from_str(raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid kubernetes resource {resource}"))?; + if let Some(name) = kubernetes_resource_name(&document) { + kept_resource_names.insert(name.to_string()); + } + projected.insert(resource.clone(), raw.clone()); + kept_resources.push(serde_yaml::Value::String(resource)); + } + *resources = kept_resources; + + if let Some(generators) = root + .get_mut(yaml_string("secretGenerator")) + .and_then(serde_yaml::Value::as_sequence_mut) + { + generators.retain(|generator| { + generator + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + != Some(KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME) + }); + } + + if let Some(replacements) = root + .get_mut(yaml_string("replacements")) + .and_then(serde_yaml::Value::as_sequence_mut) + { + replacements.retain_mut(|replacement| { + let Some(targets) = replacement + .as_mapping_mut() + .and_then(|mapping| mapping.get_mut(yaml_string("targets"))) + .and_then(serde_yaml::Value::as_sequence_mut) + else { + return false; + }; + targets.retain(|target| { + target + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("select"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|select| select.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|name| kept_resource_names.contains(name)) + }); + !targets.is_empty() + }); + } + + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {kustomization_path}"))?; + projected.insert(kustomization_path.to_string(), rendered); + Ok(projected) +} + +pub(super) fn project_kubernetes_dynamic_child_destroy_artifact_files( + artifact_files: &BTreeMap, +) -> Result> { + let kustomization_path = "kustomization.yaml"; + let raw = artifact_files.get(kustomization_path).ok_or_else(|| { + miette::miette!("dynamic kubernetes artifact snapshot is missing {kustomization_path}") + })?; + let mut document: serde_yaml::Value = serde_yaml::from_str(raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid kustomization {kustomization_path}"))?; + let root = document.as_mapping_mut().ok_or_else(|| { + miette::miette!("kustomization {kustomization_path} is not a YAML mapping") + })?; + let resources = root + .get_mut(yaml_string("resources")) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!("kustomization {kustomization_path} is missing a resources sequence") + })?; + let mut projected = artifact_files + .iter() + .filter(|(path, _)| !path.ends_with(".yaml") && path.as_str() != kustomization_path) + .map(|(path, contents)| (path.clone(), contents.clone())) + .collect::>(); + let shared_paths = [ + KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH, + KUBERNETES_PROVISIONER_ROLE_PATH, + KUBERNETES_PROVISIONER_ROLEBINDING_PATH, + KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH, + ] + .into_iter() + .collect::>(); + let mut kept_resources = Vec::new(); + for resource in resources + .iter() + .filter_map(serde_yaml::Value::as_str) + .map(str::to_owned) + { + if shared_paths.contains(resource.as_str()) { + continue; + } + let raw = artifact_files + .get(&resource) + .ok_or_else(|| miette::miette!("dynamic kubernetes artifact is missing {resource}"))?; + projected.insert(resource.clone(), raw.clone()); + kept_resources.push(serde_yaml::Value::String(resource)); + } + *resources = kept_resources; + + if let Some(generators) = root + .get_mut(yaml_string("secretGenerator")) + .and_then(serde_yaml::Value::as_sequence_mut) + { + generators.retain(|generator| { + generator + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + != Some(KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME) + }); + } + + if let Some(replacements) = root + .get_mut(yaml_string("replacements")) + .and_then(serde_yaml::Value::as_sequence_mut) + { + replacements.clear(); + } + + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {kustomization_path}"))?; + projected.insert(kustomization_path.to_string(), rendered); + Ok(projected) +} + +pub(super) fn read_artifact_snapshot(root: &Path) -> Result> { + walk_files(root)? + .into_iter() + .map(|path| { + let relative = path + .strip_prefix(root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to relativize {}", path.display()))?; + let relative = path_to_forward_slash_string(relative); + let contents = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + Ok((relative, contents)) + }) + .collect() +} + +fn rewrite_dynamic_kubernetes_apply_bundle( + artifact_root: &Path, + component_ids: &[usize], +) -> Result<()> { + let files = read_artifact_snapshot(artifact_root)?; + let projected = project_kubernetes_dynamic_child_artifact_files(&files, component_ids)?; + replace_artifact_snapshot(artifact_root, &projected) +} + +pub(super) fn prepare_dynamic_kubernetes_child_artifact( + plan: &SiteControllerRuntimePlan, + runtime_spec: &LocalChildRuntimeSpec, + artifact_root: &Path, + existing_site_peer_identities: &BTreeMap, + live_components: &BTreeMap, +) -> Result<()> { + project_dynamic_child_mesh_scope(artifact_root, Some(&plan.mesh_scope))?; + let plan_path = artifact_root.join("mesh-provision-plan.json"); + let mesh_plan = read_embedded_kubernetes_mesh_provision_plan(artifact_root)?; + let router_mesh_port = router_mesh_port_from_plan(&mesh_plan, "kubernetes")?; + let assigned = runtime_spec + .assigned_components + .iter() + .map(String::as_str) + .collect::>(); + let mut kept_component_ids = BTreeSet::new(); + let mut router_target = None; + let mut overlay_targets = Vec::with_capacity(mesh_plan.targets.len()); + let mut provision_targets = Vec::new(); + for target in mesh_plan.targets { + match target.kind { + MeshProvisionTargetKind::Component => { + if assigned.contains(target.config.identity.id.as_str()) { + kept_component_ids.insert(target.config.identity.id.clone()); + provision_targets.push(target.clone()); + overlay_targets.push(target); + } + } + MeshProvisionTargetKind::Router => { + router_target = Some(target); + } + } + } + + let Some(mut router_target) = router_target else { + return Err(miette::miette!( + "dynamic mesh provision plan {} is missing a router target", + plan_path.display() + )); + }; + filter_dynamic_router_target(&mut router_target, &kept_component_ids); + overlay_targets.push(router_target); + let mut overlay_plan = MeshProvisionPlan { + version: mesh_plan.version.clone(), + identity_seed: mesh_plan.identity_seed.clone(), + existing_peer_identities: Vec::new(), + targets: overlay_targets, + }; + let overlay_existing_peer_identities = + required_existing_mesh_peer_identities(&overlay_plan, existing_site_peer_identities)?; + overlay_plan.existing_peer_identities = + overlay_existing_peer_identities.values().cloned().collect(); + write_json(&plan_path, &overlay_plan)?; + let mut provision_plan = MeshProvisionPlan { + version: mesh_plan.version, + identity_seed: mesh_plan.identity_seed, + existing_peer_identities: Vec::new(), + targets: provision_targets, + }; + ensure_dynamic_proxy_export_component_routes( + &mut provision_plan, + &runtime_spec.proxy_exports, + &plan.router_identity_id, + )?; + rewrite_dynamic_direct_inputs( + &mut provision_plan, + &runtime_spec.direct_inputs, + live_components, + )?; + rewrite_dynamic_routed_inputs( + &mut provision_plan, + &runtime_spec.routed_inputs, + SiteKind::Kubernetes, + &plan.router_identity_id, + Some(router_mesh_port), + )?; + let provision_existing_peer_identities = + required_existing_mesh_peer_identities(&provision_plan, existing_site_peer_identities)?; + provision_plan.existing_peer_identities = provision_existing_peer_identities + .values() + .cloned() + .collect(); + write_embedded_kubernetes_mesh_provision_plan(artifact_root, &provision_plan)?; + project_dynamic_kubernetes_proxy_export_resources( + artifact_root, + &provision_plan, + &runtime_spec.proxy_exports, + )?; + rewrite_dynamic_kubernetes_apply_bundle(artifact_root, &runtime_spec.component_ids) +} + +fn kubernetes_peer_addrs_for_artifact(artifact_root: &Path) -> Result> { + let mesh_plan = read_kubernetes_runtime_mesh_provision_plan(artifact_root)?; + mesh_plan + .targets + .iter() + .filter(|target| matches!(target.kind, MeshProvisionTargetKind::Component)) + .map(|target| { + Ok(( + target.config.identity.id.clone(), + kubernetes_component_mesh_peer_addr( + artifact_root, + &target.config.identity.id, + &target.output, + target.config.mesh_listen.port(), + )?, + )) + }) + .collect() +} + +fn build_kubernetes_route_overlay_base( + artifact_root: &Path, + assigned_components: &[String], + provider_peer_addrs: &BTreeMap, + peer_identities: &BTreeMap, +) -> Result { + let mesh_plan = read_kubernetes_runtime_mesh_provision_plan(artifact_root)?; + let kept_component_ids = assigned_components.iter().cloned().collect::>(); + let mut router_target = mesh_plan + .targets + .iter() + .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) + .cloned() + .ok_or_else(|| { + miette::miette!( + "kubernetes artifact {} is missing a router mesh target", + artifact_root.display() + ) + })?; + filter_dynamic_router_target(&mut router_target, &kept_component_ids); + for route in &mut router_target.config.inbound { + if let InboundTarget::MeshForward { + peer_id, peer_addr, .. + } = &mut route.target + && let Some(resolved) = provider_peer_addrs.get(peer_id) + { + *peer_addr = resolved.clone(); + } + } + let peers = router_target + .config + .peers + .iter() + .map(|peer| { + peer_identities.get(&peer.id).map(|identity| MeshPeer { + id: identity.id.clone(), + public_key: identity.public_key, + }) + }) + .collect::>>() + .ok_or_else(|| { + miette::miette!( + "kubernetes artifact {} is missing a live mesh identity for one of its route peers", + artifact_root.display() + ) + })?; + Ok(StoredRouteOverlayPayload { + peers, + inbound_routes: router_target.config.inbound, + }) +} + +pub(super) fn write_kubernetes_live_route_overlay_payload( + artifact_root: &Path, + assigned_components: &[String], + proxy_exports: &BTreeMap, + routed_inputs: &[DynamicInputRouteRecord], + peer_identities: &BTreeMap, +) -> Result<()> { + let provider_peer_addrs = kubernetes_peer_addrs_for_artifact(artifact_root)?; + let mut payload = build_kubernetes_route_overlay_base( + artifact_root, + assigned_components, + &provider_peer_addrs, + peer_identities, + )?; + augment_route_overlay_payload( + &mut payload, + proxy_exports, + routed_inputs, + &provider_peer_addrs, + peer_identities, + None, + false, + )?; + write_dynamic_route_overlay_payload(artifact_root, &payload) +} + +fn kubernetes_network_policy_paths_by_component_label( + artifact_root: &Path, +) -> Result> { + let netpol_root = artifact_root.join("05-networkpolicies"); + if !netpol_root.is_dir() { + return Ok(BTreeMap::new()); + } + let mut netpol_paths = BTreeMap::new(); + for path in walk_files(&netpol_root)? { + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid kubernetes network policy {}", path.display()))?; + let Some(root) = document.as_mapping() else { + continue; + }; + if root + .get(yaml_string("kind")) + .and_then(serde_yaml::Value::as_str) + != Some("NetworkPolicy") + { + continue; + } + let Some(component_label) = root + .get(yaml_string("metadata")) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|metadata| metadata.get(yaml_string("labels"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|labels| labels.get(yaml_string("amber.io/component-id"))) + .and_then(serde_yaml::Value::as_str) + else { + continue; + }; + netpol_paths.insert(component_label.to_string(), path); + } + Ok(netpol_paths) +} + +fn project_dynamic_kubernetes_proxy_export_resources( + artifact_root: &Path, + mesh_plan: &MeshProvisionPlan, + proxy_exports: &BTreeMap, +) -> Result<()> { + if proxy_exports.is_empty() { + return Ok(()); + } + + let netpol_paths = kubernetes_network_policy_paths_by_component_label(artifact_root)?; + let exported_mesh_ports = proxy_exports + .values() + .map(|export| { + let component_target = mesh_plan + .targets + .iter() + .find(|target| { + matches!(target.kind, MeshProvisionTargetKind::Component) + && target.config.identity.id == export.component + }) + .ok_or_else(|| { + miette::miette!( + "dynamic proxy export provider {} is missing from the kubernetes mesh plan", + export.component + ) + })?; + Ok(( + format!("c{}", export.component_id), + component_target.config.mesh_listen.port(), + )) + }) + .collect::>>()?; + + for (component_label, mesh_port) in exported_mesh_ports { + let path = netpol_paths.get(&component_label).ok_or_else(|| { + miette::miette!( + "dynamic proxy export provider {component_label} is missing a kubernetes network \ + policy in {}", + artifact_root.join("05-networkpolicies").display() + ) + })?; + ensure_kubernetes_network_policy_router_ingress(path, mesh_port)?; + } + + Ok(()) +} + +fn ensure_kubernetes_network_policy_router_ingress(path: &Path, mesh_port: u16) -> Result<()> { + let raw = fs::read_to_string(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid kubernetes network policy {}", path.display()))?; + let root = document.as_mapping_mut().ok_or_else(|| { + miette::miette!( + "kubernetes network policy {} is not a YAML mapping", + path.display() + ) + })?; + let spec = root + .get_mut(yaml_string("spec")) + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or_else(|| { + miette::miette!( + "kubernetes network policy {} is missing a spec mapping", + path.display() + ) + })?; + let ingress = spec + .entry(yaml_string("ingress")) + .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) + .as_sequence_mut() + .ok_or_else(|| { + miette::miette!( + "kubernetes network policy {} has non-sequence spec.ingress", + path.display() + ) + })?; + + let router_peer = kubernetes_router_network_policy_peer(); + let mut matched_port_rule = false; + let mut changed = false; + for rule in ingress.iter_mut() { + let Some(rule_mapping) = rule.as_mapping_mut() else { + continue; + }; + let matches_port = rule_mapping + .get(yaml_string("ports")) + .and_then(serde_yaml::Value::as_sequence) + .is_some_and(|ports| { + ports + .iter() + .any(|port| network_policy_port_matches(port, mesh_port)) + }); + if !matches_port { + continue; + } + matched_port_rule = true; + let Some(from) = rule_mapping + .get_mut(yaml_string("from")) + .and_then(serde_yaml::Value::as_sequence_mut) + else { + break; + }; + if from.iter().any(network_policy_peer_is_router) { + break; + } + from.push(router_peer.clone()); + changed = true; + break; + } + + if !matched_port_rule { + ingress.push(serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter( + [ + ( + yaml_string("from"), + serde_yaml::Value::Sequence(vec![router_peer.clone()]), + ), + ( + yaml_string("ports"), + serde_yaml::Value::Sequence(vec![serde_yaml::Value::Mapping( + serde_yaml::Mapping::from_iter([ + (yaml_string("protocol"), yaml_string("TCP")), + ( + yaml_string("port"), + serde_yaml::Value::Number(u64::from(mesh_port).into()), + ), + ]), + )]), + ), + ], + ))); + changed = true; + } + + if !changed { + return Ok(()); + } + + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +fn kubernetes_router_network_policy_peer() -> serde_yaml::Value { + serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( + yaml_string("podSelector"), + serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( + yaml_string("matchLabels"), + serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( + yaml_string("amber.io/component"), + yaml_string(KUBERNETES_ROUTER_COMPONENT_NAME), + )])), + )])), + )])) +} + +fn network_policy_port_matches(port: &serde_yaml::Value, expected_port: u16) -> bool { + port.as_mapping() + .and_then(|port| port.get(yaml_string("port"))) + .and_then(|value| { + value + .as_u64() + .or_else(|| value.as_i64().and_then(|value| u64::try_from(value).ok())) + }) + == Some(u64::from(expected_port)) +} + +fn network_policy_peer_is_router(peer: &serde_yaml::Value) -> bool { + peer.as_mapping() + .and_then(|peer| peer.get(yaml_string("podSelector"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|selector| selector.get(yaml_string("matchLabels"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|labels| labels.get(yaml_string("amber.io/component"))) + .and_then(serde_yaml::Value::as_str) + == Some(KUBERNETES_ROUTER_COMPONENT_NAME) +} + +pub(super) fn dynamic_proxy_export_kubernetes_peer_addr( + artifact_root: &Path, + mesh_plan: &MeshProvisionPlan, + export: &DynamicProxyExportRecord, +) -> Result { + let component_target = mesh_plan + .targets + .iter() + .find(|target| { + matches!(target.kind, MeshProvisionTargetKind::Component) + && target.config.identity.id == export.component + }) + .ok_or_else(|| { + miette::miette!( + "dynamic proxy export provider {} is missing from the kubernetes mesh plan", + export.component + ) + })?; + kubernetes_component_mesh_peer_addr( + artifact_root, + &export.component, + &component_target.output, + component_target.config.mesh_listen.port(), + ) +} + +pub(super) fn project_dynamic_child_mesh_scope( + artifact_root: &Path, + mesh_scope: Option<&str>, +) -> Result<()> { + let Some(mesh_scope) = mesh_scope else { + return Ok(()); + }; + let path = artifact_root.join("mesh-provision-plan.json"); + if path.is_file() { + let mut plan: MeshProvisionPlan = read_json(&path, "mesh provision plan")?; + let existing_scopes = mesh_provision_plan_scopes(&plan); + if !project_mesh_provision_plan_scope(&mut plan, mesh_scope) { + return Ok(()); + } + write_json(&path, &plan)?; + return rewrite_dynamic_artifact_mesh_scope_literals( + artifact_root, + &existing_scopes, + mesh_scope, + ); + } + + let compose_path = artifact_root.join("compose.yaml"); + if compose_path.is_file() { + let mut plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; + let existing_scopes = mesh_provision_plan_scopes(&plan); + if !project_mesh_provision_plan_scope(&mut plan, mesh_scope) { + return Ok(()); + } + write_embedded_compose_mesh_provision_plan(artifact_root, &plan)?; + return rewrite_dynamic_artifact_mesh_scope_literals( + artifact_root, + &existing_scopes, + mesh_scope, + ); + } + + let configmap_path = artifact_root.join(KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH); + if configmap_path.is_file() { + let mut plan = read_embedded_kubernetes_mesh_provision_plan(artifact_root)?; + let existing_scopes = mesh_provision_plan_scopes(&plan); + if !project_mesh_provision_plan_scope(&mut plan, mesh_scope) { + return Ok(()); + } + write_embedded_kubernetes_mesh_provision_plan(artifact_root, &plan)?; + return rewrite_dynamic_artifact_mesh_scope_literals( + artifact_root, + &existing_scopes, + mesh_scope, + ); + } + + Err(miette::miette!( + "dynamic artifact {} is missing a mesh provision plan", + artifact_root.display() + )) +} + +fn project_mesh_provision_plan_scope(plan: &mut MeshProvisionPlan, mesh_scope: &str) -> bool { + let mut changed = false; + for target in &mut plan.targets { + if target.config.identity.mesh_scope.as_deref() == Some(mesh_scope) { + continue; + } + target.config.identity.mesh_scope = Some(mesh_scope.to_string()); + changed = true; + } + changed +} + +fn mesh_provision_plan_scopes(plan: &MeshProvisionPlan) -> BTreeSet { + let mut scopes = BTreeSet::new(); + for target in &plan.targets { + if let Some(scope) = target.config.identity.mesh_scope.as_deref() { + scopes.insert(scope.to_string()); + } + } + for identity in &plan.existing_peer_identities { + if let Some(scope) = identity.mesh_scope.as_deref() { + scopes.insert(scope.to_string()); + } + } + scopes +} + +fn rewrite_dynamic_artifact_mesh_scope_literals( + artifact_root: &Path, + existing_scopes: &BTreeSet, + mesh_scope: &str, +) -> Result<()> { + let rewrite_scopes = existing_scopes + .iter() + .filter(|scope| scope.as_str() != mesh_scope) + .cloned() + .collect::>(); + if rewrite_scopes.is_empty() { + return Ok(()); + } + + for path in walk_files(artifact_root)? { + match path.extension().and_then(|extension| extension.to_str()) { + Some("json") => rewrite_json_scope_literals(&path, &rewrite_scopes, mesh_scope)?, + Some("yaml" | "yml") => { + rewrite_yaml_scope_literals(&path, &rewrite_scopes, mesh_scope)? + } + Some("env") => rewrite_env_scope_literals(&path, &rewrite_scopes, mesh_scope)?, + _ => {} + } + } + Ok(()) +} + +fn rewrite_json_scope_literals( + path: &Path, + rewrite_scopes: &BTreeSet, + mesh_scope: &str, +) -> Result<()> { + let raw = fs::read_to_string(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_json::Value = serde_json::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid json {}", path.display()))?; + if !rewrite_scope_json_value(&mut document, rewrite_scopes, mesh_scope) { + return Ok(()); + } + let rendered = serde_json::to_string_pretty(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +fn rewrite_yaml_scope_literals( + path: &Path, + rewrite_scopes: &BTreeSet, + mesh_scope: &str, +) -> Result<()> { + let raw = fs::read_to_string(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid yaml {}", path.display()))?; + if !rewrite_scope_yaml_value(&mut document, rewrite_scopes, mesh_scope) { + return Ok(()); + } + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +fn rewrite_env_scope_literals( + path: &Path, + rewrite_scopes: &BTreeSet, + mesh_scope: &str, +) -> Result<()> { + let raw = fs::read_to_string(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut changed = false; + let mut rendered = raw + .lines() + .map(|line| { + let Some((name, value)) = line.split_once('=') else { + return line.to_string(); + }; + if !rewrite_scopes.contains(value) { + return line.to_string(); + } + changed = true; + format!("{name}={mesh_scope}") + }) + .collect::>() + .join("\n"); + if !changed { + return Ok(()); + } + if raw.ends_with('\n') { + rendered.push('\n'); + } + fs::write(path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +fn rewrite_scope_json_value( + value: &mut serde_json::Value, + rewrite_scopes: &BTreeSet, + mesh_scope: &str, +) -> bool { + match value { + serde_json::Value::String(string) => { + rewrite_scope_string_value(string, rewrite_scopes, mesh_scope) + } + serde_json::Value::Array(values) => { + let mut changed = false; + for value in values { + changed |= rewrite_scope_json_value(value, rewrite_scopes, mesh_scope); + } + changed + } + serde_json::Value::Object(map) => { + let mut changed = false; + for value in map.values_mut() { + changed |= rewrite_scope_json_value(value, rewrite_scopes, mesh_scope); + } + changed + } + _ => false, + } +} + +fn rewrite_scope_yaml_value( + value: &mut serde_yaml::Value, + rewrite_scopes: &BTreeSet, + mesh_scope: &str, +) -> bool { + match value { + serde_yaml::Value::String(string) => { + rewrite_scope_string_value(string, rewrite_scopes, mesh_scope) + } + serde_yaml::Value::Sequence(values) => { + let mut changed = false; + for value in values { + changed |= rewrite_scope_yaml_value(value, rewrite_scopes, mesh_scope); + } + changed + } + serde_yaml::Value::Mapping(map) => { + let mut changed = false; + for (_, value) in map.iter_mut() { + changed |= rewrite_scope_yaml_value(value, rewrite_scopes, mesh_scope); + } + changed + } + _ => false, + } +} + +fn rewrite_scope_string_value( + string: &mut String, + rewrite_scopes: &BTreeSet, + mesh_scope: &str, +) -> bool { + if rewrite_scopes.contains(string) { + *string = mesh_scope.to_string(); + return true; + } + let Some((name, value)) = string.split_once('=') else { + return false; + }; + if !rewrite_scopes.contains(value) { + return false; + } + *string = format!("{name}={mesh_scope}"); + true +} + +fn load_kubernetes_mesh_secret_payload( + plan: &SiteControllerRuntimePlan, + name: &str, + namespace: Option<&str>, +) -> Result<(String, KubernetesSecretPayload)> { + let namespace = namespace + .or(plan.kubernetes_namespace.as_deref()) + .ok_or_else(|| { + miette::miette!( + "kubernetes site `{}` is missing its namespace", + plan.site_id + ) + })? + .to_string(); + let output = kubectl_command(plan.context.as_deref()) + .arg("-n") + .arg(&namespace) + .arg("get") + .arg("secret") + .arg(name) + .arg("-o") + .arg("json") + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to read kubernetes mesh secret {} in namespace {}", + name, namespace + ) + })?; + if !output.status.success() { + return Err(miette::miette!( + "failed to read kubernetes mesh secret {} in namespace {}: {}", + name, + namespace, + String::from_utf8_lossy(&output.stderr).trim() + )); + } + let payload: KubernetesSecretPayload = + serde_json::from_slice(&output.stdout).map_err(|err| { + miette::miette!( + "invalid kubernetes secret payload for {} in namespace {}: {err}", + name, + namespace + ) + })?; + Ok((namespace, payload)) +} + +fn decode_kubernetes_mesh_secret_json( + payload: &KubernetesSecretPayload, + namespace: &str, + name: &str, + key: &str, + description: &str, +) -> Result +where + T: for<'de> Deserialize<'de>, +{ + let encoded = payload.data.get(key).ok_or_else(|| { + miette::miette!( + "kubernetes mesh secret {} in namespace {} is missing {}", + name, + namespace, + key + ) + })?; + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded.as_bytes()) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to decode kubernetes {description} {} in namespace {}", + name, namespace + ) + })?; + serde_json::from_slice(&bytes).map_err(|err| { + miette::miette!( + "invalid kubernetes {description} {} in namespace {}: {err}", + name, + namespace + ) + }) +} + +pub(super) fn load_kubernetes_mesh_identity_secret( + plan: &SiteControllerRuntimePlan, + name: &str, + namespace: Option<&str>, +) -> Result { + let (namespace, payload) = load_kubernetes_mesh_secret_payload(plan, name, namespace)?; + decode_kubernetes_mesh_secret_json( + &payload, + &namespace, + name, + MESH_IDENTITY_FILENAME, + "mesh identity", + ) +} + +fn load_kubernetes_mesh_config_public( + plan: &SiteControllerRuntimePlan, + name: &str, + namespace: Option<&str>, +) -> Result { + let (namespace, payload) = load_kubernetes_mesh_secret_payload(plan, name, namespace)?; + decode_kubernetes_mesh_secret_json( + &payload, + &namespace, + name, + MESH_CONFIG_FILENAME, + "mesh config", + ) +} + +pub(super) fn reconcile_site_proxy_metadata( + site_artifact_root: &Path, + site_artifact_files: &BTreeMap, +) -> Result<()> { + let Some(proxy_metadata) = site_artifact_files.get("amber-proxy.json") else { + return Ok(()); + }; + let path = site_artifact_root.join("amber-proxy.json"); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", parent.display()))?; + } + fs::write(&path, proxy_metadata) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} diff --git a/runtime/site-controller/src/default_runtime/site_runtime_support.rs b/runtime/site-controller/src/default_runtime/site_runtime_support.rs new file mode 100644 index 00000000..9de31ab0 --- /dev/null +++ b/runtime/site-controller/src/default_runtime/site_runtime_support.rs @@ -0,0 +1,1235 @@ +use super::*; + +pub(super) fn remove_dir_if_exists(path: &Path) -> Result<()> { + if !path.exists() { + return Ok(()); + } + fs::remove_dir_all(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to remove {}", path.display())) +} + +pub fn prepare_kubernetes_artifact_namespace( + run_id: &str, + site_id: &str, + artifact_dir: &Path, +) -> Result { + let namespace = kubernetes_namespace_name(run_id, site_id); + let kustomization = artifact_dir.join("kustomization.yaml"); + let contents = fs::read_to_string(&kustomization) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", kustomization.display()))?; + + let mut saw_namespace = false; + let mut out = String::new(); + for line in contents.lines() { + if line.trim_start().starts_with("namespace:") { + saw_namespace = true; + out.push_str("namespace: "); + out.push_str(&namespace); + } else { + out.push_str(line); + } + out.push('\n'); + } + if !saw_namespace { + if !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); + } + out.push_str("namespace: "); + out.push_str(&namespace); + out.push('\n'); + } + if out != contents { + fs::write(&kustomization, out) + .into_diagnostic() + .wrap_err_with(|| format!("failed to update {}", kustomization.display()))?; + } + + Ok(namespace) +} + +pub(super) fn site_supervisor_plan_for_site_runtime( + plan: &SiteControllerRuntimePlan, + artifact_dir: &Path, + kubernetes_namespace: Option, +) -> SiteSupervisorPlan { + SiteSupervisorPlan { + schema: SITE_PLAN_SCHEMA.to_string(), + version: SITE_PLAN_VERSION, + run_id: plan.run_id.clone(), + mesh_scope: plan.mesh_scope.clone(), + run_root: plan.run_root.clone(), + coordinator_pid: 0, + site_id: plan.site_id.clone(), + kind: plan.kind, + artifact_dir: artifact_dir.display().to_string(), + site_state_root: plan.site_state_root.clone(), + storage_root: plan.storage_root.clone(), + runtime_root: plan.runtime_root.clone(), + router_mesh_port: plan.router_mesh_port, + compose_project: plan.compose_project.clone(), + kubernetes_namespace, + context: plan.context.clone(), + port_forward_mesh_port: None, + port_forward_control_port: None, + observability_endpoint: plan.observability_endpoint.clone(), + site_controller_plan_path: None, + site_controller_url: None, + controller_route_ports: Vec::new(), + launch_env: plan.launch_env.clone(), + } +} + +pub(super) fn prepare_kubernetes_artifact_for_apply( + plan: &SiteControllerRuntimePlan, + artifact_dir: &Path, +) -> Result { + debug_assert_eq!(plan.kind, SiteKind::Kubernetes); + let namespace = + prepare_kubernetes_artifact_namespace(&plan.run_id, &plan.site_id, artifact_dir)?; + patch_site_artifacts( + artifact_dir, + &plan.run_id, + &plan.site_id, + plan.kind, + &plan.launch_env, + plan.observability_endpoint.as_deref(), + )?; + Ok(site_supervisor_plan_for_site_runtime( + plan, + artifact_dir, + Some(namespace), + )) +} + +pub(super) fn kubernetes_namespace_name(run_id: &str, site_id: &str) -> String { + let raw = format!("amber-{run_id}-{site_id}"); + let mut out = String::with_capacity(raw.len().min(63)); + let mut last_was_dash = false; + + for ch in raw.chars() { + let next = if ch.is_ascii_alphanumeric() { + ch.to_ascii_lowercase() + } else { + '-' + }; + if next == '-' { + if out.is_empty() || last_was_dash { + continue; + } + last_was_dash = true; + } else { + last_was_dash = false; + } + out.push(next); + if out.len() == 63 { + break; + } + } + while out.ends_with('-') { + out.pop(); + } + if out.is_empty() { + "amber".to_string() + } else { + out + } +} + +pub(super) async fn wait_for_compose_services_running( + compose_project: &str, + artifact_root: &Path, + services: &[String], + timeout: Duration, +) -> Result<()> { + if services.is_empty() { + return Ok(()); + } + let deadline = Instant::now() + timeout; + loop { + let output = compose_command(Some(compose_project), artifact_root) + .arg("ps") + .arg("--services") + .arg("--status") + .arg("running") + .args(services) + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to query compose child services in {}", + artifact_root.display() + ) + })?; + if output.status.success() { + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + let running = stdout + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .collect::>(); + if services + .iter() + .all(|service| running.contains(service.as_str())) + { + return Ok(()); + } + } + if Instant::now() >= deadline { + return Err(miette::miette!( + "timed out waiting for compose child services {:?} in {}", + services, + artifact_root.display() + )); + } + sleep(Duration::from_millis(200)).await; + } +} + +pub fn walk_files(root: &Path) -> Result> { + let mut files = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(path) = stack.pop() { + for entry in fs::read_dir(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to list {}", path.display()))? + { + let entry = entry.into_diagnostic()?; + let entry_path = entry.path(); + if entry_path.is_dir() { + stack.push(entry_path); + } else { + files.push(entry_path); + } + } + } + Ok(files) +} + +pub fn observability_endpoint_for_site(kind: SiteKind, endpoint: &str) -> Result { + if !matches!(kind, SiteKind::Compose | SiteKind::Kubernetes) { + return Ok(endpoint.to_string()); + } + + let mut url = Url::parse(endpoint) + .into_diagnostic() + .wrap_err_with(|| format!("invalid observability endpoint {endpoint}"))?; + let should_rewrite = url.host_str().is_some_and(|host| { + host.eq_ignore_ascii_case("localhost") + || host + .parse::() + .map(|addr| addr.is_loopback() || addr.is_unspecified()) + .unwrap_or(false) + }); + if should_rewrite { + let host = container_host_for_consumer(SiteKind::Direct, kind); + url.set_host(Some(&host)) + .map_err(|_| miette::miette!("failed to rewrite observability endpoint {endpoint}"))?; + } + Ok(url.to_string()) +} + +pub(crate) fn reserve_loopback_port() -> Result { + let listener = TcpListener::bind(("127.0.0.1", 0)) + .into_diagnostic() + .wrap_err("failed to allocate a loopback port")?; + Ok(listener.local_addr().into_diagnostic()?.port()) +} + +pub(super) fn site_supervisor_plan_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("site-supervisor-plan.json") +} + +pub(super) fn desired_links_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("desired-links.json") +} + +pub(super) fn empty_desired_link_state() -> DesiredLinkState { + DesiredLinkState { + schema: DESIRED_LINKS_SCHEMA.to_string(), + version: DESIRED_LINKS_VERSION, + external_slots: BTreeMap::new(), + export_peers: Vec::new(), + external_slot_overlays: BTreeMap::new(), + export_peer_overlays: BTreeMap::new(), + } +} + +pub(super) fn update_desired_overlay_for_consumer( + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExternalSlotOverlay, +) -> Result<()> { + let path = desired_links_path(site_state_root); + let mut state: DesiredLinkState = if path.is_file() { + read_json(&path, "desired links")? + } else { + empty_desired_link_state() + }; + state + .external_slot_overlays + .insert(overlay_id.to_string(), overlay); + write_json(&path, &state) +} + +pub(super) fn update_desired_overlay_for_provider( + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExportPeerOverlay, +) -> Result<()> { + let path = desired_links_path(site_state_root); + let mut state: DesiredLinkState = if path.is_file() { + read_json(&path, "desired links")? + } else { + empty_desired_link_state() + }; + state + .export_peer_overlays + .insert(overlay_id.to_string(), overlay); + write_json(&path, &state) +} + +pub(super) fn clear_desired_overlay_for_consumer( + site_state_root: &Path, + overlay_id: &str, +) -> Result<()> { + let path = desired_links_path(site_state_root); + let mut state: DesiredLinkState = if path.is_file() { + read_json(&path, "desired links")? + } else { + return Ok(()); + }; + state.external_slot_overlays.remove(overlay_id); + write_json(&path, &state) +} + +pub(super) fn clear_desired_overlay_for_provider( + site_state_root: &Path, + overlay_id: &str, +) -> Result<()> { + let path = desired_links_path(site_state_root); + let mut state: DesiredLinkState = if path.is_file() { + read_json(&path, "desired links")? + } else { + return Ok(()); + }; + state.export_peer_overlays.remove(overlay_id); + write_json(&path, &state) +} + +pub(super) fn patch_site_artifacts( + artifact_dir: &Path, + run_id: &str, + site_id: &str, + kind: SiteKind, + launch_env: &BTreeMap, + observability_endpoint: Option<&str>, +) -> Result<()> { + if matches!(kind, SiteKind::Compose) { + assign_compose_egress_network_subnets(artifact_dir, run_id, site_id)?; + } + if matches!(kind, SiteKind::Kubernetes) { + for env_file_name in [ + DEFAULT_EXTERNAL_ENV_FILE, + "component-sidecar.env", + "root-config.env", + "root-config-secret.env", + ] { + let env_file = artifact_dir.join(env_file_name); + if env_file.is_file() { + patch_generated_env_file(&env_file, launch_env)?; + } + } + + if let Some(endpoint) = observability_endpoint { + let upstream = observability_endpoint_for_site(kind, endpoint)?; + for path in walk_files(artifact_dir)? { + if path.extension().and_then(|ext| ext.to_str()) != Some("yaml") { + continue; + } + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + if !raw.contains(DEFAULT_K8S_OTEL_UPSTREAM) { + continue; + } + fs::write(&path, raw.replace(DEFAULT_K8S_OTEL_UPSTREAM, &upstream)) + .into_diagnostic() + .wrap_err_with(|| format!("failed to patch {}", path.display()))?; + } + } + } + Ok(()) +} + +pub(super) fn patch_generated_env_file( + path: &Path, + launch_env: &BTreeMap, +) -> Result<()> { + let raw = fs::read_to_string(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut out = String::new(); + for line in raw.lines() { + if let Some((key, _)) = line.split_once('=') + && let Some(value) = launch_env.get(key.trim()) + { + out.push_str(key.trim()); + out.push('='); + out.push_str(value); + out.push('\n'); + continue; + } + out.push_str(line); + out.push('\n'); + } + fs::write(path, out) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +pub(super) fn compose_command(project_name: Option<&str>, artifact_dir: &Path) -> Command { + let mut cmd = Command::new("docker"); + cmd.arg("compose") + .arg("-f") + .arg(artifact_dir.join("compose.yaml")); + if let Some(project_name) = project_name { + cmd.arg("-p").arg(project_name); + } + cmd.current_dir(artifact_dir); + cmd +} + +pub(super) fn kubectl_command(context: Option<&str>) -> Command { + let mut cmd = Command::new("kubectl"); + if let Some(context) = context { + cmd.arg("--context").arg(context); + } + cmd +} + +pub(super) fn ensure_kubernetes_namespace(plan: &SiteSupervisorPlan) -> Result<()> { + let namespace = required_str(plan.kubernetes_namespace.as_deref(), "kubernetes namespace")?; + let deadline = Instant::now() + Duration::from_secs(60); + let context = plan.context.as_deref(); + let mut last_error = None::; + loop { + let output = kubectl_command(context) + .arg("get") + .arg("namespace") + .arg(namespace) + .arg("-o") + .arg("json") + .output() + .into_diagnostic() + .wrap_err_with(|| format!("failed to query kubernetes namespace `{namespace}`"))?; + if output.status.success() { + let namespace_json: serde_json::Value = serde_json::from_slice(&output.stdout) + .into_diagnostic() + .wrap_err_with(|| format!("failed to parse kubernetes namespace `{namespace}`"))?; + let is_terminating = namespace_json + .pointer("/metadata/deletionTimestamp") + .is_some_and(|value| !value.is_null()); + if !is_terminating { + return Ok(()); + } + } else { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if stderr.contains("(NotFound)") || stderr.contains("not found") { + let create_output = kubectl_command(context) + .arg("create") + .arg("namespace") + .arg(namespace) + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to create kubernetes namespace `{namespace}`") + })?; + if create_output.status.success() { + return Ok(()); + } + last_error = Some( + String::from_utf8_lossy(&create_output.stderr) + .trim() + .to_string(), + ); + } else if !stderr.is_empty() { + last_error = Some(stderr); + } + } + if Instant::now() >= deadline { + let detail = last_error + .as_deref() + .filter(|detail| !detail.is_empty()) + .map(|detail| format!(": {detail}")) + .unwrap_or_default(); + return Err(miette::miette!( + "failed to prepare kubernetes namespace `{namespace}` within 60s{detail}" + )); + } + std::thread::sleep(Duration::from_millis(500)); + } +} + +pub(super) fn ensure_kubernetes_workloads_ready(plan: &SiteSupervisorPlan) -> Result<()> { + let namespace = required_str(plan.kubernetes_namespace.as_deref(), "kubernetes namespace")?; + let context = plan.context.as_deref(); + let expected = kubernetes_expected_workloads(Path::new(&plan.artifact_dir))?; + wait_for_named_kubernetes_resources( + context, + namespace, + "job", + "condition=complete", + &expected.jobs, + &format!("wait for kubernetes jobs for site `{}`", plan.site_id), + )?; + wait_for_named_kubernetes_resources( + context, + namespace, + "deployment", + "condition=available", + &expected.deployments, + &format!( + "wait for kubernetes deployments for site `{}`", + plan.site_id + ), + )?; + Ok(()) +} + +#[derive(Default)] +pub(super) struct KubernetesArtifactWorkloads { + pub(super) jobs: Vec, + pub(super) deployments: Vec, +} + +pub(super) fn kubernetes_expected_workloads( + artifact_dir: &Path, +) -> Result { + let mut jobs = BTreeSet::new(); + let mut deployments = BTreeSet::new(); + for path in walk_files(artifact_dir)? { + if !matches!( + path.extension().and_then(|ext| ext.to_str()), + Some("yaml" | "yml") + ) { + continue; + } + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read kubernetes artifact {}", path.display()))?; + for document in serde_yaml::Deserializer::from_str(&raw) { + use serde::Deserialize as _; + + let value = serde_yaml::Value::deserialize(document) + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to parse kubernetes artifact {}", path.display()) + })?; + let kind = value + .get("kind") + .and_then(serde_yaml::Value::as_str) + .unwrap_or_default(); + let name = value + .get("metadata") + .and_then(serde_yaml::Value::as_mapping) + .and_then(|metadata| metadata.get(serde_yaml::Value::String("name".to_string()))) + .and_then(serde_yaml::Value::as_str) + .unwrap_or_default(); + if name.is_empty() { + continue; + } + match kind { + "Job" => { + jobs.insert(name.to_string()); + } + "Deployment" => { + deployments.insert(name.to_string()); + } + _ => {} + } + } + } + Ok(KubernetesArtifactWorkloads { + jobs: jobs.into_iter().collect(), + deployments: deployments.into_iter().collect(), + }) +} + +pub(super) fn wait_for_named_kubernetes_resources( + context: Option<&str>, + namespace: &str, + resource_kind: &str, + condition: &str, + names: &[String], + label: &str, +) -> Result<()> { + for name in names { + wait_for_named_kubernetes_resource( + context, + namespace, + resource_kind, + condition, + name, + label, + )?; + } + Ok(()) +} + +pub(super) fn wait_for_named_kubernetes_resource( + context: Option<&str>, + namespace: &str, + resource_kind: &str, + condition: &str, + name: &str, + label: &str, +) -> Result<()> { + let resource = format!("{resource_kind}/{name}"); + let deadline = Instant::now() + KUBERNETES_WORKLOAD_READY_TIMEOUT; + loop { + let get_output = kubectl_command(context) + .args(["-n", namespace, "get", resource.as_str()]) + .output() + .into_diagnostic() + .wrap_err_with(|| format!("{label}: query {resource}"))?; + if !get_output.status.success() { + if Instant::now() >= deadline { + let stderr = String::from_utf8_lossy(&get_output.stderr) + .trim() + .to_string(); + let detail = if stderr.is_empty() { + format!("status {}", get_output.status) + } else { + stderr + }; + return Err(miette::miette!("{label} failed: {detail}")); + } + std::thread::sleep(Duration::from_millis(250)); + continue; + } + + let remaining = deadline.saturating_duration_since(Instant::now()); + let timeout = format!("{}s", remaining.as_secs().max(1)); + let wait_output = kubectl_command(context) + .args([ + "-n", + namespace, + "wait", + "--for", + condition, + "--timeout", + timeout.as_str(), + resource.as_str(), + ]) + .output() + .into_diagnostic() + .wrap_err_with(|| format!("{label}: wait for {resource}"))?; + if wait_output.status.success() { + return Ok(()); + } + let stderr = String::from_utf8_lossy(&wait_output.stderr) + .trim() + .to_string(); + if stderr.contains("not found") && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(250)); + continue; + } + let detail = if stderr.is_empty() { + format!("status {}", wait_output.status) + } else { + stderr + }; + return Err(miette::miette!("{label} failed: {detail}")); + } +} + +pub(super) fn mesh_protocol(protocol: NetworkProtocol) -> Result { + Ok(match protocol { + NetworkProtocol::Http | NetworkProtocol::Https => MeshProtocol::Http, + NetworkProtocol::Tcp => MeshProtocol::Tcp, + _ => { + return Err(miette::miette!( + "mixed-site mesh links do not support protocol `{protocol}`" + )); + } + }) +} + +pub(crate) fn site_ready_timeout_for_kind(kind: SiteKind) -> Duration { + match kind { + SiteKind::Kubernetes => KUBERNETES_WORKLOAD_READY_TIMEOUT + KUBERNETES_SITE_READY_BUFFER, + SiteKind::Direct | SiteKind::Compose | SiteKind::Vm => Duration::from_secs(120), + } +} + +pub(super) fn amber_cli_executable() -> Result { + if let Some(path) = env::var_os("CARGO_BIN_EXE_amber") { + let path = PathBuf::from(path); + if path.is_file() { + return Ok(path); + } + } + + let current = env::current_exe() + .into_diagnostic() + .wrap_err("failed to resolve amber executable path")?; + let exe_name = format!("amber{}", std::env::consts::EXE_SUFFIX); + for dir in [current.parent(), current.parent().and_then(Path::parent)] + .into_iter() + .flatten() + { + let candidate = dir.join(&exe_name); + if candidate.is_file() { + return Ok(candidate); + } + } + Err(miette::miette!( + "failed to locate the amber executable next to {}", + current.display() + )) +} + +pub(crate) fn spawn_detached_child( + work_dir: &Path, + log_path: &Path, + build: impl FnOnce(&mut Command), +) -> Result { + #[cfg(unix)] + use std::os::unix::process::CommandExt as _; + + let exe = amber_cli_executable()?; + let log = fs::File::create(log_path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create log {}", log_path.display()))?; + let log_err = log + .try_clone() + .into_diagnostic() + .wrap_err("failed to clone log handle")?; + let mut cmd = Command::new(exe); + cmd.current_dir(work_dir); + cmd.stdin(Stdio::null()); + cmd.stdout(Stdio::from(log)); + cmd.stderr(Stdio::from(log_err)); + #[cfg(unix)] + unsafe { + cmd.pre_exec(|| { + if libc::setsid() == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + build(&mut cmd); + cmd.spawn().into_diagnostic().wrap_err_with(|| { + format!( + "failed to spawn background amber process in {}", + work_dir.display() + ) + }) +} + +pub(super) async fn stop_child(child: &mut Child) -> Result<()> { + #[cfg(unix)] + { + terminate_recorded_processes(&[child.id()]).await?; + let _ = child.wait(); + Ok(()) + } + + #[cfg(not(unix))] + { + send_sigterm(child.id()); + let _ = wait_for_child_exit(child, PROCESS_SHUTDOWN_GRACE_PERIOD).await; + Ok(()) + } +} + +#[cfg(not(unix))] +pub(super) async fn wait_for_child_exit(child: &mut Child, timeout: Duration) -> Result<()> { + let deadline = Instant::now() + timeout; + loop { + if child.try_wait().into_diagnostic()?.is_some() { + return Ok(()); + } + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + return Ok(()); + } + sleep(Duration::from_millis(100)).await; + } +} + +pub(super) async fn wait_for_pid_exit(pid: u32, timeout: Duration) -> bool { + let deadline = Instant::now() + timeout; + loop { + if !pid_is_alive(pid) { + return true; + } + if Instant::now() >= deadline { + return false; + } + sleep(Duration::from_millis(100)).await; + } +} + +pub(super) async fn router_mesh_listener_ready(addr: SocketAddr) -> bool { + tokio::net::TcpStream::connect(addr).await.is_ok() +} + +pub(super) async fn wait_for_socket_listener(addr: SocketAddr) -> Result<()> { + let deadline = Instant::now() + Duration::from_secs(30); + while Instant::now() < deadline { + if router_mesh_listener_ready(addr).await { + return Ok(()); + } + sleep(Duration::from_millis(100)).await; + } + Err(miette::miette!("timed out waiting for listener {}", addr)) +} + +pub(super) fn endpoint_returns_http_response_blocking( + addr: SocketAddr, + timeout: Duration, +) -> Result { + let deadline = std::time::Instant::now() + timeout; + while std::time::Instant::now() < deadline { + if let Ok(mut stream) = + std::net::TcpStream::connect_timeout(&addr, Duration::from_millis(100)) + { + let _ = + stream.write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"); + let mut buf = [0u8; 32]; + if let Ok(read) = stream.read(&mut buf) + && read > 0 + && buf[..read].starts_with(b"HTTP/") + { + return Ok(true); + } + } + std::thread::sleep(Duration::from_millis(100)); + } + Ok(false) +} + +pub(super) fn endpoint_accepts_stable_connection_blocking( + addr: SocketAddr, + timeout: Duration, +) -> Result { + let deadline = std::time::Instant::now() + timeout; + while std::time::Instant::now() < deadline { + if std::net::TcpStream::connect_timeout(&addr, Duration::from_millis(100)).is_ok() { + return Ok(true); + } + std::thread::sleep(Duration::from_millis(100)); + } + Ok(false) +} + +pub(super) fn find_header_end(buf: &[u8]) -> Option { + buf.windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|index| index + 4) +} + +pub(super) fn pid_is_alive(pid: u32) -> bool { + #[cfg(unix)] + { + let alive = if unsafe { libc::kill(pid as i32, 0) } == 0 { + true + } else { + std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) + }; + alive && process_status_code(pid) != Some('Z') + } + + #[cfg(not(unix))] + { + let _ = pid; + true + } +} + +#[cfg(unix)] +fn process_status_code(pid: u32) -> Option { + let output = Command::new("ps") + .arg("-o") + .arg("stat=") + .arg("-p") + .arg(pid.to_string()) + .output() + .ok()?; + if !output.status.success() { + return None; + } + parse_process_status_code(&String::from_utf8_lossy(&output.stdout)) +} + +#[cfg(unix)] +fn parse_process_status_code(raw: &str) -> Option { + raw.split_whitespace() + .next()? + .chars() + .next() + .map(|state| state.to_ascii_uppercase()) +} + +#[cfg(unix)] +pub(super) fn send_sigterm(pid: u32) { + unsafe { + libc::kill(pid as i32, libc::SIGTERM); + } +} + +#[cfg(not(unix))] +pub(super) fn send_sigterm(_pid: u32) {} + +#[cfg(unix)] +pub(super) fn send_signal_to_pids(pids: &[u32], signal: i32) { + for pid in pids { + unsafe { + libc::kill(*pid as i32, signal); + } + } +} + +#[cfg(not(unix))] +pub(super) fn send_signal_to_pids(_pids: &[u32], _signal: i32) {} + +#[cfg(unix)] +pub(super) fn send_signal_to_process_group(root_pid: u32, signal: i32) { + unsafe { + libc::kill(-(root_pid as i32), signal); + } +} + +#[cfg(not(unix))] +pub(super) fn send_signal_to_process_group(_root_pid: u32, _signal: i32) {} + +pub(super) fn process_tree_postorder(root_pid: u32) -> Result> { + Ok(vec![root_pid]) +} + +pub(super) async fn terminate_recorded_processes(root_pids: &[u32]) -> Result<()> { + for pid in root_pids { + send_sigterm(*pid); + if !wait_for_pid_exit(*pid, PROCESS_SHUTDOWN_GRACE_PERIOD).await { + #[cfg(unix)] + unsafe { + libc::kill(*pid as i32, libc::SIGKILL); + } + } + } + Ok(()) +} + +pub(crate) async fn resolve_link_external_url_for_output( + provider: &LaunchedSite, + provider_output_dir: &Path, + link: &RunLink, + consumer_kind: SiteKind, + run_root: &Path, + bridge_proxies: &mut BTreeMap, +) -> Result { + if !link_needs_bridge_proxy(provider.receipt.kind, consumer_kind) { + return external_slot_url(provider, provider_output_dir, link, consumer_kind); + } + + let port = ensure_bridge_proxy( + run_root, + provider, + provider_output_dir, + &link.export_name, + consumer_kind, + bridge_proxies, + ) + .await?; + bridge_proxy_external_url(port, link.protocol, consumer_kind) +} + +pub(super) fn link_needs_bridge_proxy(provider_kind: SiteKind, consumer_kind: SiteKind) -> bool { + matches!(consumer_kind, SiteKind::Compose | SiteKind::Kubernetes) + && provider_kind != SiteKind::Kubernetes +} + +pub(super) async fn ensure_bridge_proxy( + run_root: &Path, + provider: &LaunchedSite, + provider_output_dir: &Path, + export_name: &str, + consumer_kind: SiteKind, + bridge_proxies: &mut BTreeMap, +) -> Result { + let key = BridgeProxyKey { + provider_output_dir: provider_output_dir.display().to_string(), + export_name: export_name.to_string(), + consumer_kind, + }; + if let Some(proxy) = bridge_proxies.get_mut(&key) + && proxy.child.try_wait().into_diagnostic()?.is_none() + { + return Ok(proxy.listen.port()); + } + + let listen = bridge_proxy_bind_addr(consumer_kind, reserve_loopback_port()?); + let child = spawn_bridge_proxy(run_root, provider, provider_output_dir, export_name, listen)?; + wait_for_socket_listener(bridge_proxy_probe_addr(listen)).await?; + bridge_proxies.insert(key, BridgeProxyHandle { child, listen }); + Ok(listen.port()) +} + +pub(super) fn spawn_bridge_proxy( + run_root: &Path, + provider: &LaunchedSite, + provider_output_dir: &Path, + export_name: &str, + listen: SocketAddr, +) -> Result { + let logs_root = run_root.join("bridge-proxies"); + fs::create_dir_all(&logs_root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create {}", logs_root.display()))?; + let log_path = logs_root.join(format!("{export_name}.log")); + spawn_detached_child(run_root, &log_path, |cmd| { + cmd.arg("proxy") + .arg(provider_output_dir) + .arg("--export") + .arg(bridge_proxy_export_binding(export_name, listen)); + if provider.receipt.kind == SiteKind::Kubernetes { + let control = provider.router_control.to_string(); + cmd.arg("--router-addr") + .arg(provider.router_addr.to_string()) + .arg("--router-control-addr") + .arg(control); + } + }) +} + +pub(super) fn bridge_proxy_export_binding(export_name: &str, listen: SocketAddr) -> String { + format!("{export_name}={}:{}", listen.ip(), listen.port()) +} + +pub(super) fn bridge_proxy_bind_addr(consumer_kind: SiteKind, port: u16) -> SocketAddr { + host_service_bind_addr_for_consumer(consumer_kind, port) +} + +pub(super) fn bridge_proxy_probe_addr(listen: SocketAddr) -> SocketAddr { + listener_probe_addr(listen) +} + +pub(super) fn bridge_proxy_external_url( + port: u16, + protocol: NetworkProtocol, + consumer_kind: SiteKind, +) -> Result { + let host = host_service_host_for_consumer(consumer_kind); + Ok(match protocol { + NetworkProtocol::Http | NetworkProtocol::Https => format!("http://{host}:{port}"), + NetworkProtocol::Tcp => format!("tcp://{host}:{port}"), + _ => { + return Err(miette::miette!( + "mixed-site bridge proxy does not support protocol `{protocol}`" + )); + } + }) +} + +pub(crate) fn host_service_host_for_consumer(consumer_kind: SiteKind) -> String { + match consumer_kind { + SiteKind::Compose => CONTAINER_HOST_ALIAS.to_string(), + SiteKind::Direct | SiteKind::Vm | SiteKind::Kubernetes => { + container_host_for_consumer(SiteKind::Direct, consumer_kind) + } + } +} + +pub(super) fn consumer_needs_host_wide_listener(consumer_kind: SiteKind) -> bool { + matches!(consumer_kind, SiteKind::Compose | SiteKind::Kubernetes) +} + +pub fn host_service_bind_addr_for_consumer(consumer_kind: SiteKind, port: u16) -> SocketAddr { + host_proxy_bind_addr(consumer_needs_host_wide_listener(consumer_kind), port) +} + +pub fn router_mesh_addr_for_consumer( + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, +) -> Result { + match consumer_kind { + SiteKind::Compose | SiteKind::Kubernetes => { + let addr = router_mesh_addr + .parse::() + .into_diagnostic() + .wrap_err_with(|| { + format!("invalid live router mesh address `{router_mesh_addr}`") + })?; + let host = container_host_for_consumer(provider_kind, consumer_kind); + Ok(format!("{host}:{}", addr.port())) + } + SiteKind::Direct | SiteKind::Vm => { + #[cfg(target_os = "linux")] + { + Ok(rewrite_peer_addr_for_slirp_gateway(router_mesh_addr)) + } + #[cfg(not(target_os = "linux"))] + { + Ok(router_mesh_addr.to_string()) + } + } + } +} + +pub(super) fn host_proxy_bind_addr(needs_host_wide_listener: bool, port: u16) -> SocketAddr { + if needs_host_wide_listener { + SocketAddr::from(([0, 0, 0, 0], port)) + } else { + SocketAddr::from(([127, 0, 0, 1], port)) + } +} + +pub(super) fn listener_probe_addr(listen: SocketAddr) -> SocketAddr { + if listen.ip().is_unspecified() { + SocketAddr::from(([127, 0, 0, 1], listen.port())) + } else { + listen + } +} + +pub(super) fn external_slot_url( + provider: &LaunchedSite, + provider_output_dir: &Path, + link: &RunLink, + consumer_kind: SiteKind, +) -> Result { + let host = container_host_for_consumer(provider.receipt.kind, consumer_kind); + let route_id = provider_export_route_id(provider_output_dir, link)?; + let mut mesh_url = Url::parse(&format!("mesh://{}:{}", host, provider.router_addr.port())) + .into_diagnostic() + .wrap_err("failed to build mesh link url")?; + let peer_key = + base64::engine::general_purpose::STANDARD.encode(provider.router_identity.public_key); + mesh_url + .query_pairs_mut() + .append_pair("peer_id", &provider.router_identity.id) + .append_pair("peer_key", &peer_key) + .append_pair("route_id", &route_id) + .append_pair("capability", &link.export_name); + Ok(mesh_url.to_string()) +} + +pub(super) fn provider_export_route_id( + provider_output_dir: &Path, + link: &RunLink, +) -> Result { + if let Some(route_id) = load_output_proxy_metadata(provider_output_dir)? + .exports + .get(&link.export_name) + .and_then(|export| export.route_id.clone()) + { + return Ok(route_id); + } + Ok(router_export_route_id( + &link.export_name, + mesh_protocol(link.protocol)?, + )) +} + +pub(super) fn container_host_for_consumer( + provider_kind: SiteKind, + consumer_kind: SiteKind, +) -> String { + let container_host_ip = container_host_ip(); + container_host_from_resolved_ip(provider_kind, consumer_kind, container_host_ip.as_deref()) +} + +pub(super) fn container_host_from_resolved_ip( + provider_kind: SiteKind, + consumer_kind: SiteKind, + container_host_ip: Option<&str>, +) -> String { + match consumer_kind { + SiteKind::Direct | SiteKind::Vm => "127.0.0.1".to_string(), + SiteKind::Compose => { + if provider_kind == SiteKind::Kubernetes { + container_host_ip + .unwrap_or(CONTAINER_HOST_ALIAS) + .to_string() + } else { + CONTAINER_HOST_ALIAS.to_string() + } + } + SiteKind::Kubernetes => container_host_ip + .unwrap_or(CONTAINER_HOST_ALIAS) + .to_string(), + } +} + +pub(super) fn container_host_ip() -> Option { + KUBERNETES_CONTAINER_HOST_IP + .get_or_init(resolve_container_host_ip) + .clone() +} + +pub(super) fn resolve_container_host_ip() -> Option { + if cfg!(target_os = "linux") { + return resolve_linux_container_host_ip(); + } + resolve_desktop_container_host_ip() +} + +pub(super) fn resolve_linux_container_host_ip() -> Option { + let output = Command::new("docker") + .arg("network") + .arg("inspect") + .arg("bridge") + .arg("--format") + .arg("{{(index .IPAM.Config 0).Gateway}}") + .output() + .ok()?; + if !output.status.success() { + return None; + } + + let host = String::from_utf8(output.stdout).ok()?.trim().to_string(); + host.parse::().ok()?; + Some(host) +} + +pub(super) fn resolve_desktop_container_host_ip() -> Option { + let output = Command::new("docker") + .arg("run") + .arg("--rm") + .arg("busybox:1.36.1") + .arg("nslookup") + .arg(CONTAINER_HOST_ALIAS) + .output() + .ok()?; + if !output.status.success() { + return None; + } + + String::from_utf8(output.stdout) + .ok()? + .lines() + .flat_map(str::split_whitespace) + .filter_map(|token| token.parse::().ok()) + .map(|ip| ip.to_string()) + .next_back() +} + +#[cfg(test)] +mod tests { + #[cfg(unix)] + use super::parse_process_status_code; + + #[cfg(unix)] + #[test] + fn parse_process_status_code_normalizes_unix_ps_output() { + assert_eq!(parse_process_status_code("S+\n"), Some('S')); + assert_eq!(parse_process_status_code("z\n"), Some('Z')); + assert_eq!(parse_process_status_code(""), None); + } +} diff --git a/runtime/site-controller/src/dynamic_caps.rs b/runtime/site-controller/src/dynamic_caps.rs index 59493af4..3909e26d 100644 --- a/runtime/site-controller/src/dynamic_caps.rs +++ b/runtime/site-controller/src/dynamic_caps.rs @@ -77,6 +77,8 @@ pub(crate) enum DynamicCapabilityShareOutcome { #[derive(Clone, Debug)] pub(crate) struct DynamicCapabilityRevokeOutcome { + // Production code only needs to know whether revocation changed anything; tests assert the + // exact grant set to keep cascade revocation behavior pinned down. #[cfg_attr(not(test), allow(dead_code))] pub(crate) revoked_grant_ids: Vec, } @@ -157,6 +159,16 @@ pub(crate) struct ControlDynamicResolveOriginRequest { pub(crate) source: DynamicCapabilityControlSourceRequest, } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct InternalDynamicResolveOriginRequest { + pub(crate) holder_component_id: String, + #[serde(flatten)] + pub(crate) source: DynamicCapabilityControlSourceRequest, + pub(crate) holder_peer_id: String, + pub(crate) holder_peer_key_b64: String, + pub(crate) holder_site_kind: SiteKind, +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct ControlDynamicResolveOriginResponse { pub(crate) held_id: String, diff --git a/runtime/site-controller/src/lib.rs b/runtime/site-controller/src/lib.rs index 123b6454..6cc19a67 100644 --- a/runtime/site-controller/src/lib.rs +++ b/runtime/site-controller/src/lib.rs @@ -63,6 +63,7 @@ mod api; mod ccs_api; mod control_state_api; mod control_state_mcp; +mod default_runtime; mod dynamic_caps; mod http; mod mcp; @@ -76,20 +77,36 @@ mod state; mod tests; pub use self::{ + default_runtime::{ + SiteControllerPeerRouterRoute, add_compose_router_published_route_ports, + assign_compose_egress_network_subnets, cleanup_dynamic_site_children, + host_service_bind_addr_for_consumer, inject_compose_site_controller, + inject_kubernetes_site_controller, inject_site_controller_peer_router_routes, + observability_endpoint_for_site, prepare_kubernetes_artifact_namespace, + router_mesh_addr_for_consumer, set_compose_router_published_mesh_port, + set_site_artifact_mesh_identity_seed, walk_files, + }, runtime_api::{ - DesiredExportPeerOverlay, DesiredExternalSlotOverlay, LaunchedSite, - LiveComponentRuntimeMetadata, SharedSiteControllerRuntime, SiteControllerRuntime, - SiteControllerRuntimeFuture, SiteControllerRuntimePlan, SiteReceipt, - launched_site_from_receipt, parse_control_endpoint, + DesiredExportPeerOverlay, DesiredExternalSlotOverlay, DesiredRouteOverlay, LaunchedSite, + LiveComponentRuntimeMetadata, SiteControllerRuntimeFuture, SiteControllerRuntimePlan, + SiteReceipt, launched_site_from_receipt, parse_control_endpoint, project_kubernetes_dynamic_child_artifact_files, site_controller_plan_path, site_controller_runtime_child_root_for_site, site_controller_runtime_plan_from_controller_plan, site_state_path, }, - site_controller::run_site_controller, state::{ - DynamicInputRouteRecord, DynamicInputRouteTarget, DynamicProxyExportRecord, - DynamicSitePlanRecord, SiteControllerPeerPlan, SiteControllerPlan, - authority_url_for_listen_addr, build_site_controller_state, generate_framework_auth_token, - write_control_state, write_site_controller_plan, + DynamicInputDirectRecord, DynamicInputRouteRecord, DynamicInputRouteTarget, + DynamicProxyExportRecord, SITE_CONTROLLER_INTERNAL_CAPABILITY, SITE_CONTROLLER_PORT, + SITE_CONTROLLER_SERVICE_NAME, SiteControllerPlan, authority_url_for_listen_addr, + build_site_controller_state, generate_framework_auth_token, + site_controller_internal_route_id, write_control_state, write_site_controller_plan, }, }; + +pub async fn run_site_controller_default(plan_path: PathBuf) -> Result<()> { + site_controller::run_site_controller( + plan_path, + default_runtime::default_site_controller_runtime(), + ) + .await +} diff --git a/runtime/site-controller/src/orchestration.rs b/runtime/site-controller/src/orchestration.rs index 3a5ffde1..b8052233 100644 --- a/runtime/site-controller/src/orchestration.rs +++ b/runtime/site-controller/src/orchestration.rs @@ -5,36 +5,10 @@ use amber_mesh::{ InboundRoute, InboundTarget, MeshConfigPublic, MeshPeer, OutboundRoute, router_external_route_id, }; +use amber_proxy::{ControlEndpoint, fetch_router_identity}; use super::{http::*, planner::*, state::*, *}; -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct SiteControllerPrepareRequest { - pub(crate) site_plan: DynamicSitePlanRecord, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct SiteControllerPublishRequest { - pub(crate) site_plan: DynamicSitePlanRecord, -} - -#[derive(Clone, Debug, Serialize, Deserialize, Default)] -pub(crate) struct SiteControllerDestroyRequest { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) desired_site_plan: Option, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(super) struct ControlCreateChildRequest { - pub(super) cap_instance_id: String, - pub(super) request: CreateChildRequest, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(super) struct ControlDestroyChildRequest { - pub(super) cap_instance_id: String, -} - #[derive(Debug)] pub(super) struct ProtocolApiError(pub(super) ProtocolErrorResponse); @@ -114,6 +88,58 @@ impl IntoResponse for ProtocolApiError { } } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ResolveExternalLinkUrlRequest { + pub(crate) child_id: u64, + pub(crate) link: RunLink, + pub(crate) consumer_kind: SiteKind, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ResolveExternalLinkUrlResponse { + pub(crate) external_url: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct PublishExternalSlotOverlayRequest { + pub(crate) overlay_id: String, + pub(crate) slot_name: String, + pub(crate) url: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ClearExternalSlotOverlayRequest { + pub(crate) overlay_id: String, + pub(crate) slot_name: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct PublishExportPeerOverlayRequest { + pub(crate) overlay_id: String, + pub(crate) export_name: String, + pub(crate) peer_id: String, + pub(crate) peer_key_b64: String, + pub(crate) protocol: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) route_id: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ClearExportPeerOverlayRequest { + pub(crate) overlay_id: String, + pub(crate) export_name: String, + pub(crate) peer_id: String, + pub(crate) peer_key_b64: String, + pub(crate) protocol: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) route_id: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct RevokeDynamicCapabilityOriginOverlaysRequest { + pub(crate) overlay_ids: Vec, +} + pub(super) fn control_state_step_error( step: &str, err: impl std::fmt::Display, @@ -136,14 +162,6 @@ pub(super) fn controller_protocol_error( ) } -pub(super) fn site_state_root_for(app: &ControlStateApp, site_id: &str) -> PathBuf { - Path::new(&app.state_root).join(site_id) -} - -pub(super) fn site_controller_plan_path_for_site(app: &ControlStateApp, site_id: &str) -> PathBuf { - site_controller_plan_path(&site_state_root_for(app, site_id)) -} - pub(super) fn site_receipt_from_manager_state(state: &SiteManagerStateView) -> SiteReceipt { SiteReceipt { kind: state.kind, @@ -163,128 +181,648 @@ pub(super) fn site_receipt_from_manager_state(state: &SiteManagerStateView) -> S } } -pub(super) fn full_site_plan_record( +pub(super) fn load_site_manager_state( + app: &ControlStateApp, site_id: &str, - site_plan: &amber_compiler::run_plan::RunSitePlan, -) -> DynamicSitePlanRecord { - DynamicSitePlanRecord { - site_id: site_id.to_string(), - kind: site_plan.site.kind, - router_identity_id: site_plan.router_identity_id.clone(), - component_ids: site_plan - .scenario_ir - .components +) -> std::result::Result { + let state_path = site_state_path(&app.state_root, site_id); + if state_path.is_file() { + return read_json(&state_path, "site manager state").map_err(|err| { + protocol_error( + ProtocolErrorCode::SiteNotActive, + &format!("site `{site_id}` manager state is unavailable: {err}"), + ) + }); + } + if site_id == app.controller_plan.site_id { + return Ok(local_site_manager_state_view(app)); + } + Err(protocol_error( + ProtocolErrorCode::SiteNotActive, + &format!( + "site `{site_id}` manager state is unavailable: {}", + state_path.display() + ), + )) +} + +fn local_site_manager_state_view(app: &ControlStateApp) -> SiteManagerStateView { + let runtime_plan = site_controller_runtime_plan_from_controller_plan(&app.controller_plan); + let router_identity = app + .runtime + .load_live_site_router_mesh_config(&runtime_plan) + .ok() + .map(|router_mesh| { + ( + router_mesh.identity.id, + base64::engine::general_purpose::STANDARD.encode(router_mesh.identity.public_key), + ) + }); + SiteManagerStateView { + status: "running".to_string(), + kind: app.controller_plan.kind, + artifact_dir: app.controller_plan.artifact_dir.clone(), + supervisor_pid: 0, + process_pid: None, + compose_project: app.controller_plan.compose_project.clone(), + kubernetes_namespace: app.controller_plan.kubernetes_namespace.clone(), + port_forward_pid: None, + context: app.controller_plan.context.clone(), + router_control: app.controller_plan.local_router_control.clone(), + router_mesh_addr: app.controller_plan.published_router_mesh_addr.clone(), + router_identity_id: router_identity.as_ref().map(|(id, _)| id.clone()), + router_public_key_b64: router_identity.map(|(_, public_key_b64)| public_key_b64), + site_controller_pid: None, + site_controller_url: Some(app.controller_plan.authority_url.clone()), + } +} + +pub(super) fn load_launched_site( + app: &ControlStateApp, + site_id: &str, +) -> std::result::Result { + let state = load_site_manager_state(app, site_id)?; + if state.status != "running" { + return Err(protocol_error( + ProtocolErrorCode::SiteNotActive, + &format!("site `{site_id}` is not running"), + )); + } + let receipt = site_receipt_from_manager_state(&state); + launched_site_from_receipt(&receipt, &app.mesh_scope).map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("site `{site_id}` routing metadata is invalid: {err}"), + ) + }) +} + +fn framework_route_overlay_id(authority_site_id: &str) -> String { + format!("framework-component-routes:{authority_site_id}") +} + +fn framework_component_route( + record: &CapabilityInstanceRecord, + allowed_issuers: Vec, + target: InboundTarget, +) -> InboundRoute { + InboundRoute { + route_id: record.route_id.clone(), + capability: record.capability.clone(), + capability_kind: Some("component".to_string()), + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target, + allowed_issuers, + } +} + +fn framework_controller_external_target() -> InboundTarget { + InboundTarget::External { + url_env: amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV.to_string(), + optional: false, + } +} + +async fn peer_router_identity_for_overlay( + app: &ControlStateApp, + site_id: &str, +) -> std::result::Result { + let site_app = SiteControllerApp { + control: app.clone(), + router_auth_token: app.control_state_auth_token.clone(), + }; + let identity = super::site_controller::peer_router_identity_via_router(&site_app, site_id) + .await + .map_err(|err| err.0)?; + Ok(MeshPeer { + id: identity.id, + public_key: identity.public_key, + }) +} + +fn peer_router_mesh_addr_for_overlay( + app: &ControlStateApp, + site_id: &str, +) -> std::result::Result { + app.controller_plan + .peer_router_mesh_addrs + .get(site_id) + .cloned() + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "site controller `{}` has no mesh address for peer site `{site_id}`", + app.controller_plan.site_id + ), + ) + }) +} + +fn local_router_control_endpoint( + app: &ControlStateApp, +) -> std::result::Result { + if let Some(raw) = app.controller_plan.local_router_control.as_deref() { + return parse_control_endpoint(raw).map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("local router control endpoint is invalid: {err}"), + ) + }); + } + let state = load_site_manager_state(app, &app.controller_plan.site_id)?; + let raw = state.router_control.ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + "local router control endpoint is unavailable", + ) + })?; + parse_control_endpoint(&raw).map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("local router control endpoint is invalid: {err}"), + ) + }) +} + +pub(super) async fn local_router_identity_for_overlay( + app: &ControlStateApp, +) -> std::result::Result { + let state = load_site_manager_state(app, &app.controller_plan.site_id)?; + if let (Some(id), Some(public_key_b64)) = + (state.router_identity_id, state.router_public_key_b64) + && let Ok(decoded) = + base64::engine::general_purpose::STANDARD.decode(public_key_b64.as_bytes()) + && let Ok(public_key) = decoded.as_slice().try_into() + { + return Ok(MeshPeer { id, public_key }); + } + if let Ok(router_mesh) = app.runtime.load_live_site_router_mesh_config( + &site_controller_runtime_plan_from_controller_plan(&app.controller_plan), + ) { + return Ok(MeshPeer { + id: router_mesh.identity.id, + public_key: router_mesh.identity.public_key, + }); + } + let identity = fetch_router_identity(&local_router_control_endpoint(app)?) + .await + .map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("failed to read local router identity: {err}"), + ) + })?; + Ok(MeshPeer { + id: identity.id, + public_key: identity.public_key, + }) +} + +fn local_site_state_root(app: &ControlStateApp) -> &Path { + Path::new(&app.controller_plan.site_state_root) +} + +fn local_site_kind_from_state( + state: &FrameworkControlState, + site_id: &str, +) -> std::result::Result { + state + .placement + .offered_sites + .get(site_id) + .map(|site| site.kind) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("site `{site_id}` is missing from offered sites"), + ) + }) +} + +pub(super) async fn resolve_external_link_url_local( + app: &ControlStateApp, + request: &ResolveExternalLinkUrlRequest, +) -> std::result::Result { + if request.link.provider_site != app.controller_plan.site_id { + return Err(controller_protocol_error( + ProtocolErrorCode::PublishFailed, + &request.link.provider_site, + "resolve provider link url", + format!( + "controller `{}` only resolves provider surfaces for its own site", + app.controller_plan.site_id + ), + )); + } + let child = { + let state = app.control_state.lock().await; + cloned_child_record(&state, request.child_id)? + }; + let provider = load_launched_site(app, &app.controller_plan.site_id)?; + let provider_output_dir = provider_output_dir_for_link( + app, + &child, + Path::new(&provider.receipt.artifact_dir), + &request.link, + ); + let external_url = app + .runtime + .resolve_link_external_url( + &provider, + &provider_output_dir, + &request.link, + request.consumer_kind, + &app.run_root, + ) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::PublishFailed, + &request.link.provider_site, + "compute external slot overlay", + err, + ) + })?; + Ok(ResolveExternalLinkUrlResponse { external_url }) +} + +pub(super) async fn publish_external_slot_overlay_local( + app: &ControlStateApp, + request: &PublishExternalSlotOverlayRequest, +) -> std::result::Result<(), ProtocolErrorResponse> { + register_external_slot_with_retry( + &local_router_control_endpoint(app)?, + &request.slot_name, + &request.url, + Duration::from_secs(30), + ) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::PublishFailed, + &app.controller_plan.site_id, + "publish external slot overlay", + err, + ) + })?; + app.runtime + .update_desired_overlay_for_consumer( + local_site_state_root(app), + &request.overlay_id, + DesiredExternalSlotOverlay { + slot_name: request.slot_name.clone(), + url: request.url.clone(), + }, + ) + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &app.controller_plan.site_id, + "persist desired external slot overlay", + err, + ) + }) +} + +pub(super) async fn clear_external_slot_overlay_local( + app: &ControlStateApp, + request: &ClearExternalSlotOverlayRequest, +) -> std::result::Result<(), ProtocolErrorResponse> { + app.runtime + .clear_desired_overlay_for_consumer(local_site_state_root(app), &request.overlay_id) + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &app.controller_plan.site_id, + "persist external slot overlay removal", + err, + ) + })?; + clear_external_slot_with_retry( + &local_router_control_endpoint(app)?, + &request.slot_name, + Duration::from_secs(30), + ) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &app.controller_plan.site_id, + "retract external slot overlay", + err, + ) + }) +} + +pub(super) async fn publish_export_peer_overlay_local( + app: &ControlStateApp, + request: &PublishExportPeerOverlayRequest, +) -> std::result::Result<(), ProtocolErrorResponse> { + register_export_peer_with_retry( + &local_router_control_endpoint(app)?, + &request.export_name, + &request.peer_id, + &request.peer_key_b64, + &request.protocol, + request.route_id.as_deref(), + Duration::from_secs(30), + ) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::PublishFailed, + &app.controller_plan.site_id, + "publish export-peer overlay", + err, + ) + })?; + app.runtime + .update_desired_overlay_for_provider( + local_site_state_root(app), + &request.overlay_id, + DesiredExportPeerOverlay { + export_name: request.export_name.clone(), + peer_id: request.peer_id.clone(), + peer_key_b64: request.peer_key_b64.clone(), + protocol: request.protocol.clone(), + route_id: request.route_id.clone(), + }, + ) + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &app.controller_plan.site_id, + "persist desired export-peer overlay", + err, + ) + }) +} + +pub(super) async fn clear_export_peer_overlay_local( + app: &ControlStateApp, + request: &ClearExportPeerOverlayRequest, +) -> std::result::Result<(), ProtocolErrorResponse> { + app.runtime + .clear_desired_overlay_for_provider(local_site_state_root(app), &request.overlay_id) + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &app.controller_plan.site_id, + "persist export-peer overlay removal", + err, + ) + })?; + unregister_export_peer_with_retry( + &local_router_control_endpoint(app)?, + &request.export_name, + &request.peer_id, + &request.peer_key_b64, + &request.protocol, + request.route_id.as_deref(), + Duration::from_secs(30), + ) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &app.controller_plan.site_id, + "retract export-peer overlay", + err, + ) + }) +} + +pub(super) async fn revoke_dynamic_capability_origin_overlays_local( + app: &ControlStateApp, + request: &RevokeDynamicCapabilityOriginOverlaysRequest, +) -> std::result::Result<(), ProtocolErrorResponse> { + let endpoint = local_router_control_endpoint(app)?; + for overlay_id in &request.overlay_ids { + revoke_route_overlay_with_retry(&endpoint, overlay_id, Duration::from_secs(30)) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &app.controller_plan.site_id, + "retract dynamic capability origin overlays", + err, + ) + })?; + } + Ok(()) +} + +pub(super) async fn framework_route_overlay_payload( + app: &ControlStateApp, +) -> std::result::Result, ProtocolErrorResponse> { + let state = app.control_state.lock().await.clone(); + let local_site_id = app.controller_plan.site_id.clone(); + let mut overlay = DesiredRouteOverlay { + peers: Vec::new(), + inbound_routes: Vec::new(), + }; + + for record in state.capability_instances.values() { + let authority_site_id = site_id_for_authority_realm(&state, record.authority_realm_id)?; + let consumer_site_id = record.recipient_site_id.clone(); + + if authority_site_id == local_site_id { + if consumer_site_id == local_site_id { + overlay.inbound_routes.push(framework_component_route( + record, + vec![record.recipient_peer_id.clone()], + framework_controller_external_target(), + )); + continue; + } + let consumer_peer = peer_router_identity_for_overlay(app, &consumer_site_id).await?; + if !overlay.peers.iter().any(|peer| peer.id == consumer_peer.id) { + overlay.peers.push(consumer_peer.clone()); + } + overlay.inbound_routes.push(framework_component_route( + record, + vec![consumer_peer.id], + framework_controller_external_target(), + )); + continue; + } + + if consumer_site_id != local_site_id { + continue; + } + + let authority_peer = peer_router_identity_for_overlay(app, &authority_site_id).await?; + if !overlay + .peers .iter() - .map(|component| component.id) - .collect(), - assigned_components: site_plan.assigned_components.clone(), - artifact_files: site_plan.artifact_files.clone(), - desired_artifact_files: site_plan.artifact_files.clone(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), + .any(|peer| peer.id == authority_peer.id) + { + overlay.peers.push(authority_peer.clone()); + } + overlay.inbound_routes.push(framework_component_route( + record, + vec![record.recipient_peer_id.clone()], + InboundTarget::MeshForward { + peer_addr: peer_router_mesh_addr_for_overlay(app, &authority_site_id)?, + peer_id: authority_peer.id, + route_id: record.route_id.clone(), + capability: record.capability.clone(), + }, + )); } -} -pub(super) fn desired_site_plan_map( - state: &FrameworkControlState, - site_ids: &BTreeSet, -) -> std::result::Result, ProtocolErrorResponse> { - let planned = build_site_plan_subset( - &decode_live_scenario(state)?, - &placement_file_from_state(state), - &run_plan_activation_from_state(state), - &live_assignment_map(state), - site_ids, - ) - .map_err(|err| { - protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &format!("failed to rebuild desired site plans for affected sites: {err}"), - ) - })?; - Ok(planned - .iter() - .map(|(site_id, site_plan)| (site_id.clone(), full_site_plan_record(site_id, site_plan))) - .collect()) + if overlay.peers.is_empty() && overlay.inbound_routes.is_empty() { + return Ok(None); + } + overlay.peers.sort_by(|left, right| left.id.cmp(&right.id)); + overlay + .inbound_routes + .sort_by(|left, right| left.route_id.cmp(&right.route_id)); + Ok(Some(overlay)) } -pub(super) fn load_site_manager_state( +pub(super) async fn reconcile_local_framework_routes( app: &ControlStateApp, - site_id: &str, -) -> std::result::Result { - read_json( - &site_state_path(&app.state_root, site_id), - "site manager state", - ) - .map_err(|err| { - protocol_error( - ProtocolErrorCode::SiteNotActive, - &format!("site `{site_id}` manager state is unavailable: {err}"), +) -> std::result::Result<(), ProtocolErrorResponse> { + let overlay_id = framework_route_overlay_id(&app.controller_plan.site_id); + let endpoint = local_router_control_endpoint(app)?; + if let Some(overlay) = framework_route_overlay_payload(app).await? { + apply_route_overlay_with_retry( + &endpoint, + &overlay_id, + &overlay.peers, + &overlay.inbound_routes, + Duration::from_secs(30), ) - }) -} - -pub(super) fn load_launched_site( - app: &ControlStateApp, - site_id: &str, -) -> std::result::Result { - let state = load_site_manager_state(app, site_id)?; - if state.status != "running" { - return Err(protocol_error( - ProtocolErrorCode::SiteNotActive, - &format!("site `{site_id}` is not running"), - )); + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &app.controller_plan.site_id, + "publish framework route overlay", + err, + ) + })?; + } else { + revoke_route_overlay_with_retry(&endpoint, &overlay_id, Duration::from_secs(30)) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &app.controller_plan.site_id, + "retract framework route overlay", + err, + ) + })?; } - let receipt = site_receipt_from_manager_state(&state); - launched_site_from_receipt(&receipt, &app.mesh_scope).map_err(|err| { - protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &format!("site `{site_id}` routing metadata is invalid: {err}"), - ) - }) + + Ok(()) } -pub(super) fn load_site_controller_plan( +fn peer_site_router_url( app: &ControlStateApp, site_id: &str, -) -> std::result::Result { +) -> std::result::Result { if site_id == app.controller_plan.site_id { - return Ok(app.controller_plan.as_ref().clone()); + return Ok(app.controller_plan.authority_url.clone()); } - let path = site_controller_plan_path_for_site(app, site_id); - read_json(&path, "site controller plan").map_err(|err| { - protocol_error( - ProtocolErrorCode::SiteNotActive, - &format!("site `{site_id}` controller plan is unavailable: {err}"), - ) - }) + app.controller_plan + .peer_site_router_urls + .get(site_id) + .cloned() + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::OriginUnavailable, + &format!( + "site controller `{}` has no router forward path to peer site `{site_id}`", + app.controller_plan.site_id + ), + ) + }) } -pub(super) fn load_site_runtime_plan( +async fn peer_controller_post_json_via_router Deserialize<'de>>( app: &ControlStateApp, site_id: &str, -) -> std::result::Result { - Ok(site_controller_runtime_plan_from_controller_plan( - &load_site_controller_plan(app, site_id)?, + path: &str, + body: &TReq, + code: ProtocolErrorCode, + action: &str, +) -> std::result::Result { + let response = app + .client + .post(format!( + "{}{}", + peer_site_router_url(app, site_id)?.trim_end_matches('/'), + path + )) + .header(super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER, "1") + .json(body) + .send() + .await + .map_err(|err| controller_protocol_error(code, site_id, action, err))?; + if response.status().is_success() { + return response + .json() + .await + .map_err(|err| controller_protocol_error(code, site_id, action, err)); + } + let status = response.status(); + let body = response + .bytes() + .await + .map_err(|err| controller_protocol_error(code, site_id, action, err))?; + if let Ok(protocol_error) = serde_json::from_slice::(&body) { + return Err(protocol_error); + } + Err(controller_protocol_error( + code, + site_id, + action, + format!("peer controller returned {status}"), )) } -fn controller_authority_url<'a>( - app: &'a ControlStateApp, +async fn peer_controller_post_no_content_via_router( + app: &ControlStateApp, site_id: &str, -) -> std::result::Result<&'a str, ProtocolErrorResponse> { - if site_id == app.controller_plan.site_id { - return Ok(app.controller_plan.authority_url.as_str()); + path: &str, + body: &TReq, + code: ProtocolErrorCode, + action: &str, +) -> std::result::Result<(), ProtocolErrorResponse> { + let response = app + .client + .post(format!( + "{}{}", + peer_site_router_url(app, site_id)?.trim_end_matches('/'), + path + )) + .header(super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER, "1") + .json(body) + .send() + .await + .map_err(|err| controller_protocol_error(code, site_id, action, err))?; + if response.status().is_success() { + return Ok(()); } - app.peer_controllers - .get(site_id) - .map(|controller| controller.authority_url.as_str()) - .ok_or_else(|| { - protocol_error( - ProtocolErrorCode::SiteNotActive, - &format!("site `{site_id}` controller metadata is unavailable"), - ) - }) + let status = response.status(); + let body = response + .bytes() + .await + .map_err(|err| controller_protocol_error(code, site_id, action, err))?; + if let Ok(protocol_error) = serde_json::from_slice::(&body) { + return Err(protocol_error); + } + Err(controller_protocol_error( + code, + site_id, + action, + format!("peer controller returned {status}"), + )) } pub(super) fn load_site_runtime_plan_at( @@ -330,14 +868,25 @@ pub(super) async fn publish_dynamic_capability_origin( request: &dynamic_caps::PublishDynamicCapabilityOriginRequest, ) -> std::result::Result { + if site_id == app.controller_plan.site_id { + return publish_dynamic_capability_origin_local( + &LocalDynamicCapabilityOriginApp { + site_state_root: PathBuf::from(&app.controller_plan.site_state_root), + runtime: app.runtime.clone(), + }, + request.clone(), + ) + .await + .map_err(|err| err.0); + } let url = format!( "{}/v1/internal/dynamic-caps/origins/publish", - controller_authority_url(app, site_id)?.trim_end_matches('/') + peer_site_router_url(app, site_id)?.trim_end_matches('/') ); let response = app .client .post(url) - .header(FRAMEWORK_AUTH_HEADER, app.control_state_auth_token.as_ref()) + .header(super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER, "1") .json(request) .send() .await @@ -345,8 +894,8 @@ pub(super) async fn publish_dynamic_capability_origin( protocol_error( ProtocolErrorCode::OriginUnavailable, &format!( - "failed to reach site controller on site `{site_id}` while publishing dynamic \ - capability origin: {err}" + "failed to reach site controller on site `{site_id}` through the site router \ + while publishing dynamic capability origin: {err}" ), ) })?; @@ -355,8 +904,8 @@ pub(super) async fn publish_dynamic_capability_origin( protocol_error( ProtocolErrorCode::OriginUnavailable, &format!( - "site controller on site `{site_id}` returned invalid JSON while publishing \ - dynamic capability origin: {err}" + "site controller on site `{site_id}` returned invalid JSON through the site \ + router while publishing dynamic capability origin: {err}" ), ) }); @@ -366,8 +915,8 @@ pub(super) async fn publish_dynamic_capability_origin( protocol_error( ProtocolErrorCode::OriginUnavailable, &format!( - "failed to read site controller error response on site `{site_id}` while \ - publishing dynamic capability origin: {err}" + "failed to read site controller error response on site `{site_id}` through the \ + site router while publishing dynamic capability origin: {err}" ), ) })?; @@ -377,60 +926,12 @@ pub(super) async fn publish_dynamic_capability_origin( Err(protocol_error( ProtocolErrorCode::OriginUnavailable, &format!( - "site controller on site `{site_id}` returned {status} while publishing dynamic \ - capability origin" + "site controller on site `{site_id}` returned {status} through the site router while \ + publishing dynamic capability origin" ), )) } -#[derive(Clone, Debug)] -pub(super) struct DynamicCapabilityRuntimeEndpoint { - pub(super) site_id: String, - pub(super) runtime: LiveComponentRuntimeMetadata, -} - -pub(super) fn dynamic_capability_component_runtime_endpoint( - app: &ControlStateApp, - state: &FrameworkControlState, - logical_component_id: &str, -) -> std::result::Result { - let moniker = dynamic_caps::moniker_from_logical_component_id(logical_component_id)?; - let assignments = live_assignment_map(state); - let site_id = assignments.get(moniker).ok_or_else(|| { - protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!("live component `{logical_component_id}` is not assigned to a live site"), - ) - })?; - let site_plan = load_site_runtime_plan(app, site_id)?; - let runtime = app - .runtime - .collect_live_component_runtime_metadata(&site_plan) - .map_err(|err| { - protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "failed to resolve live runtime metadata for component \ - `{logical_component_id}` on site `{site_id}`: {err}" - ), - ) - })? - .remove(moniker) - .ok_or_else(|| { - protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "live runtime metadata for component `{logical_component_id}` is unavailable \ - on site `{site_id}`" - ), - ) - })?; - Ok(DynamicCapabilityRuntimeEndpoint { - site_id: site_id.clone(), - runtime, - }) -} - pub(super) fn dynamic_capability_origin_route_surface( runtime: &LiveComponentRuntimeMetadata, site_components: &BTreeMap, @@ -784,152 +1285,66 @@ pub(super) async fn publish_dynamic_capability_origin_local( }) } -pub(super) async fn call_site_controller( +pub(super) async fn prepare_child_on_site( app: &ControlStateApp, - site_id: &str, - path: &str, - body: Option<&B>, - error_code: ProtocolErrorCode, - action: &str, + state: &FrameworkControlState, + child: &LiveChildRecord, ) -> std::result::Result<(), ProtocolErrorResponse> { - if site_id == app.controller_plan.site_id { - let local_child_id = || -> std::result::Result { - path.rsplit('/') - .nth(1) - .ok_or_else(|| { - controller_protocol_error( - error_code, - site_id, - action, - format!("site controller path `{path}` is malformed"), - ) - })? - .parse() - .map_err(|err| { - controller_protocol_error( - error_code, - site_id, - action, - format!("site controller child id in `{path}` is malformed: {err}"), - ) - }) - }; - match path { - path if path.ends_with("/prepare") => { - let request = serde_json::to_value(body).expect("site op body should serialize"); - let request = serde_json::from_value::(request) - .expect("prepare request should deserialize"); - return app - .runtime - .prepare_child(&app.controller_plan, local_child_id()?, request.site_plan) - .await - .map_err(|err| controller_protocol_error(error_code, site_id, action, err)); - } - path if path.ends_with("/publish") => { - let request = serde_json::to_value(body).expect("site op body should serialize"); - let request = serde_json::from_value::(request) - .expect("publish request should deserialize"); - return app - .runtime - .publish_child(&app.controller_plan, local_child_id()?, request.site_plan) - .await - .map_err(|err| controller_protocol_error(error_code, site_id, action, err)); - } - path if path.ends_with("/destroy") => { - let request = serde_json::to_value(body).expect("site op body should serialize"); - let request = serde_json::from_value::(request) - .expect("destroy request should deserialize"); - return app - .runtime - .destroy_child( - &app.controller_plan, - local_child_id()?, - request.desired_site_plan, - ) - .await - .map_err(|err| controller_protocol_error(error_code, site_id, action, err)); - } - _ => {} - } - } else { - let controller = app.peer_controllers.get(site_id).ok_or_else(|| { - controller_protocol_error( - error_code, - site_id, - action, - "site controller is not present in peer metadata", - ) - })?; - let url = format!("{}{}", controller.authority_url.trim_end_matches('/'), path); - let request = app - .client - .post(url) - .header(FRAMEWORK_AUTH_HEADER, app.control_state_auth_token.as_ref()); - let request = if let Some(body) = body { - request.json(body) - } else { - request - }; - let response = request - .send() - .await - .map_err(|err| controller_protocol_error(error_code, site_id, action, err))?; - if response.status().is_success() { - return Ok(()); - } - let status = response.status(); - let body = response.text().await.unwrap_or_default(); + let site_id = child_runtime_site_id(child)?; + if site_id != app.controller_plan.site_id { return Err(controller_protocol_error( - error_code, - site_id, - action, - format!("HTTP {status}: {}", body.trim()), + ProtocolErrorCode::PrepareFailed, + &site_id, + "prepare child", + format!( + "dynamic child `{}` targeted site `{}` but controller `{}` only creates local \ + children", + child.name, site_id, app.controller_plan.site_id + ), )); } - Err(controller_protocol_error( - error_code, - site_id, - action, - format!("unsupported site controller action path `{path}`"), - )) -} - -pub(super) async fn prepare_child_on_site( - app: &ControlStateApp, - child_id: u64, - site_plan: &DynamicSitePlanRecord, -) -> std::result::Result<(), ProtocolErrorResponse> { - let path = format!("/v1/controller/site/children/{child_id}/prepare"); - call_site_controller( - app, - &site_plan.site_id, - &path, - Some(&SiteControllerPrepareRequest { - site_plan: site_plan.clone(), - }), - ProtocolErrorCode::PrepareFailed, - "prepare child", - ) - .await + app.runtime + .prepare_child(&app.controller_plan, state.clone(), child.clone()) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::PrepareFailed, + &site_id, + "prepare child", + err, + ) + }) } pub(super) async fn publish_child_on_site( app: &ControlStateApp, - child_id: u64, - site_plan: &DynamicSitePlanRecord, + state: &FrameworkControlState, + child: &LiveChildRecord, ) -> std::result::Result<(), ProtocolErrorResponse> { - let path = format!("/v1/controller/site/children/{child_id}/publish"); - call_site_controller( - app, - &site_plan.site_id, - &path, - Some(&SiteControllerPublishRequest { - site_plan: site_plan.clone(), - }), - ProtocolErrorCode::PublishFailed, - "publish child", - ) - .await + let site_id = child_runtime_site_id(child)?; + if site_id != app.controller_plan.site_id { + return Err(controller_protocol_error( + ProtocolErrorCode::PublishFailed, + &site_id, + "publish child", + format!( + "dynamic child `{}` targeted site `{}` but controller `{}` only publishes local \ + children", + child.name, site_id, app.controller_plan.site_id + ), + )); + } + app.runtime + .publish_child(&app.controller_plan, state.clone(), child.clone()) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::PublishFailed, + &site_id, + "publish child", + err, + ) + }) } fn site_controller_child_needs_prepare(err: &ProtocolErrorResponse, child_id: u64) -> bool { @@ -941,74 +1356,65 @@ fn site_controller_child_needs_prepare(err: &ProtocolErrorResponse, child_id: u6 pub(super) async fn publish_child_on_site_with_prepare_retry( app: &ControlStateApp, - child_id: u64, - site_plan: &DynamicSitePlanRecord, + state: &FrameworkControlState, + child: &LiveChildRecord, ) -> std::result::Result<(), ProtocolErrorResponse> { - match publish_child_on_site(app, child_id, site_plan).await { + match publish_child_on_site(app, state, child).await { Ok(()) => Ok(()), - Err(err) if site_controller_child_needs_prepare(&err, child_id) => { - prepare_child_on_site(app, child_id, site_plan).await?; - publish_child_on_site(app, child_id, site_plan).await + Err(err) if site_controller_child_needs_prepare(&err, child.child_id) => { + prepare_child_on_site(app, state, child).await?; + publish_child_on_site(app, state, child).await } Err(err) => Err(err), } } -pub(super) async fn rollback_child_on_site( - app: &ControlStateApp, - child_id: u64, - site_id: &str, -) -> Result<()> { - if site_id == app.controller_plan.site_id { - return app - .runtime - .rollback_child(&app.controller_plan, child_id) - .await; - } - let path = format!("/v1/controller/site/children/{child_id}/rollback"); - let controller = app - .peer_controllers - .get(site_id) - .ok_or_else(|| miette::miette!("site `{site_id}` controller metadata is unavailable"))?; - let response = app - .client - .post(format!( - "{}{}", - controller.authority_url.trim_end_matches('/'), - path - )) - .header(FRAMEWORK_AUTH_HEADER, app.control_state_auth_token.as_ref()) - .send() - .await - .into_diagnostic() - .wrap_err_with(|| format!("failed to rollback child on site `{site_id}`"))?; - if response.status().is_success() { - return Ok(()); - } - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - Err(miette::miette!( - "failed to rollback child on site `{site_id}`: HTTP {status}: {}", - body.trim() - )) -} - -pub(super) async fn destroy_child_on_site( +pub(super) async fn rollback_child_on_site( app: &ControlStateApp, child_id: u64, site_id: &str, - desired_site_plan: Option, +) -> Result<()> { + if site_id != app.controller_plan.site_id { + return Err(miette::miette!( + "dynamic child plan targeted site `{site_id}` but controller `{}` only rolls back \ + local children", + app.controller_plan.site_id + )); + } + app.runtime + .rollback_child(&app.controller_plan, child_id) + .await +} + +pub(super) async fn destroy_child_on_site( + app: &ControlStateApp, + state: &FrameworkControlState, + child: &LiveChildRecord, ) -> std::result::Result<(), ProtocolErrorResponse> { - let path = format!("/v1/controller/site/children/{child_id}/destroy"); - call_site_controller( - app, - site_id, - &path, - Some(&SiteControllerDestroyRequest { desired_site_plan }), - ProtocolErrorCode::ControlStateUnavailable, - "destroy child", - ) - .await + let site_id = child_runtime_site_id(child)?; + if site_id != app.controller_plan.site_id { + return Err(controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &site_id, + "destroy child", + format!( + "dynamic child `{}` targeted site `{site_id}` but controller `{}` only destroys \ + local children", + child.name, app.controller_plan.site_id + ), + )); + } + app.runtime + .destroy_child(&app.controller_plan, state.clone(), child.clone()) + .await + .map_err(|err| { + controller_protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &site_id, + "destroy child", + err, + ) + }) } pub(super) async fn publish_external_slot_overlay( @@ -1019,60 +1425,49 @@ pub(super) async fn publish_external_slot_overlay( let overlay_id = overlay_id_for_link_action(child, link, |action| { matches!(action, DynamicOverlayAction::ExternalSlot { .. }) })?; - let provider = load_launched_site(app, &link.provider_site)?; - let consumer = load_launched_site(app, &link.consumer_site)?; - let provider_output_dir = - provider_output_dir_for_link(app, child, Path::new(&provider.receipt.artifact_dir), link); - let external_url = app - .runtime - .resolve_link_external_url( - &provider, - &provider_output_dir, - link, - consumer.receipt.kind, - &app.run_root, - ) - .await - .map_err(|err| { - controller_protocol_error( - ProtocolErrorCode::PublishFailed, - &link.consumer_site, - "compute external slot overlay", - err, - ) - })?; - register_external_slot_with_retry( - &consumer.router_control, - &link.external_slot_name, - &external_url, - Duration::from_secs(30), - ) - .await - .map_err(|err| { - controller_protocol_error( + let consumer_kind = { + let state = app.control_state.lock().await; + local_site_kind_from_state(&state, &link.consumer_site)? + }; + let resolve_request = ResolveExternalLinkUrlRequest { + child_id: child.child_id, + link: link.clone(), + consumer_kind, + }; + let external_url = if link.provider_site == app.controller_plan.site_id { + resolve_external_link_url_local(app, &resolve_request) + .await? + .external_url + } else { + peer_controller_post_json_via_router::<_, ResolveExternalLinkUrlResponse>( + app, + &link.provider_site, + "/v1/internal/link-overlays/external-url", + &resolve_request, ProtocolErrorCode::PublishFailed, + "resolve provider link url", + ) + .await? + .external_url + }; + let publish_request = PublishExternalSlotOverlayRequest { + overlay_id: overlay_id.to_string(), + slot_name: link.external_slot_name.clone(), + url: external_url, + }; + if link.consumer_site == app.controller_plan.site_id { + publish_external_slot_overlay_local(app, &publish_request).await + } else { + peer_controller_post_no_content_via_router( + app, &link.consumer_site, + "/v1/internal/link-overlays/external-slot/publish", + &publish_request, + ProtocolErrorCode::PublishFailed, "publish external slot overlay", - err, ) - })?; - app.runtime - .update_desired_overlay_for_consumer( - &site_state_root_for(app, &link.consumer_site), - overlay_id, - DesiredExternalSlotOverlay { - slot_name: link.external_slot_name.clone(), - url: external_url, - }, - ) - .map_err(|err| { - controller_protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &link.consumer_site, - "persist desired external slot overlay", - err, - ) - }) + .await + } } pub(super) async fn publish_export_peer_overlay( @@ -1083,49 +1478,34 @@ pub(super) async fn publish_export_peer_overlay( let overlay_id = overlay_id_for_link_action(child, link, |action| { matches!(action, DynamicOverlayAction::ExportPeer { .. }) })?; - let provider = load_launched_site(app, &link.provider_site)?; - let consumer = load_launched_site(app, &link.consumer_site)?; - let consumer_key = - base64::engine::general_purpose::STANDARD.encode(consumer.router_identity.public_key); let route_id = export_peer_route_id(child, link)?; - register_export_peer_with_retry( - &provider.router_control, - &link.export_name, - &consumer.router_identity.id, - &consumer_key, - &link.protocol.to_string(), - Some(&route_id), - Duration::from_secs(30), - ) - .await - .map_err(|err| { - controller_protocol_error( - ProtocolErrorCode::PublishFailed, + let consumer_identity = if link.consumer_site == app.controller_plan.site_id { + local_router_identity_for_overlay(app).await? + } else { + peer_router_identity_for_overlay(app, &link.consumer_site).await? + }; + let publish_request = PublishExportPeerOverlayRequest { + overlay_id: overlay_id.to_string(), + export_name: link.export_name.clone(), + peer_id: consumer_identity.id, + peer_key_b64: base64::engine::general_purpose::STANDARD + .encode(consumer_identity.public_key), + protocol: link.protocol.to_string(), + route_id: Some(route_id), + }; + if link.provider_site == app.controller_plan.site_id { + publish_export_peer_overlay_local(app, &publish_request).await + } else { + peer_controller_post_no_content_via_router( + app, &link.provider_site, + "/v1/internal/link-overlays/export-peer/publish", + &publish_request, + ProtocolErrorCode::PublishFailed, "publish export-peer overlay", - err, - ) - })?; - app.runtime - .update_desired_overlay_for_provider( - &site_state_root_for(app, &link.provider_site), - overlay_id, - DesiredExportPeerOverlay { - export_name: link.export_name.clone(), - peer_id: consumer.router_identity.id, - peer_key_b64: consumer_key, - protocol: link.protocol.to_string(), - route_id: Some(route_id), - }, ) - .map_err(|err| { - controller_protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &link.provider_site, - "persist desired export-peer overlay", - err, - ) - }) + .await + } } pub(super) fn child_link_records(child: &LiveChildRecord) -> Vec { @@ -1187,11 +1567,8 @@ pub(super) fn provider_output_dir_for_link( if !provider_in_child { return provider_artifact_dir.to_path_buf(); } - site_controller_runtime_child_root_for_site( - &site_state_root_for(app, &link.provider_site), - child.child_id, - ) - .join("artifact") + site_controller_runtime_child_root_for_site(local_site_state_root(app), child.child_id) + .join("artifact") } pub(super) fn export_peer_route_id( @@ -1282,40 +1659,29 @@ pub(super) async fn clear_external_slot_overlay( let overlay_id = overlay_id_for_link_action(child, link, |action| { matches!(action, DynamicOverlayAction::ExternalSlot { .. }) })?; - app.runtime - .clear_desired_overlay_for_consumer( - &site_state_root_for(app, &link.consumer_site), - overlay_id, - ) - .map_err(|err| { - controller_protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &link.consumer_site, - "persist external slot overlay removal", - err, - ) - })?; { let state = app.control_state.lock().await; if link_still_required(&state, child_id, link) { return Ok(()); } } - let consumer = load_launched_site(app, &link.consumer_site)?; - clear_external_slot_with_retry( - &consumer.router_control, - &link.external_slot_name, - Duration::from_secs(30), - ) - .await - .map_err(|err| { - controller_protocol_error( - ProtocolErrorCode::ControlStateUnavailable, + let clear_request = ClearExternalSlotOverlayRequest { + overlay_id: overlay_id.to_string(), + slot_name: link.external_slot_name.clone(), + }; + if link.consumer_site == app.controller_plan.site_id { + clear_external_slot_overlay_local(app, &clear_request).await + } else { + peer_controller_post_no_content_via_router( + app, &link.consumer_site, + "/v1/internal/link-overlays/external-slot/clear", + &clear_request, + ProtocolErrorCode::ControlStateUnavailable, "retract external slot overlay", - err, ) - }) + .await + } } pub(super) async fn clear_export_peer_overlay( @@ -1327,48 +1693,40 @@ pub(super) async fn clear_export_peer_overlay( let overlay_id = overlay_id_for_link_action(child, link, |action| { matches!(action, DynamicOverlayAction::ExportPeer { .. }) })?; - app.runtime - .clear_desired_overlay_for_provider( - &site_state_root_for(app, &link.provider_site), - overlay_id, - ) - .map_err(|err| { - controller_protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &link.provider_site, - "persist export-peer overlay removal", - err, - ) - })?; { let state = app.control_state.lock().await; if link_still_required(&state, child_id, link) { return Ok(()); } } - let provider = load_launched_site(app, &link.provider_site)?; - let consumer = load_launched_site(app, &link.consumer_site)?; - let consumer_key = - base64::engine::general_purpose::STANDARD.encode(consumer.router_identity.public_key); let route_id = export_peer_route_id(child, link)?; - unregister_export_peer_with_retry( - &provider.router_control, - &link.export_name, - &consumer.router_identity.id, - &consumer_key, - &link.protocol.to_string(), - Some(&route_id), - Duration::from_secs(30), - ) - .await - .map_err(|err| { - controller_protocol_error( - ProtocolErrorCode::ControlStateUnavailable, + let consumer_identity = if link.consumer_site == app.controller_plan.site_id { + local_router_identity_for_overlay(app).await? + } else { + peer_router_identity_for_overlay(app, &link.consumer_site).await? + }; + let clear_request = ClearExportPeerOverlayRequest { + overlay_id: overlay_id.to_string(), + export_name: link.export_name.clone(), + peer_id: consumer_identity.id, + peer_key_b64: base64::engine::general_purpose::STANDARD + .encode(consumer_identity.public_key), + protocol: link.protocol.to_string(), + route_id: Some(route_id), + }; + if link.provider_site == app.controller_plan.site_id { + clear_export_peer_overlay_local(app, &clear_request).await + } else { + peer_controller_post_no_content_via_router( + app, &link.provider_site, + "/v1/internal/link-overlays/export-peer/clear", + &clear_request, + ProtocolErrorCode::ControlStateUnavailable, "retract export-peer overlay", - err, ) - }) + .await + } } pub(super) async fn publish_child_overlays( @@ -1434,114 +1792,27 @@ pub(super) async fn retract_dynamic_capability_origin_overlays( }; for (site_id, overlay_ids) in overlays_by_site { - let router_control = load_site_manager_state(app, &site_id)? - .router_control - .ok_or_else(|| { - controller_protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &site_id, - "retract dynamic capability origin overlays", - "site router control endpoint is unavailable", - ) - })?; - let endpoint = parse_control_endpoint(&router_control).map_err(|err| { - controller_protocol_error( - ProtocolErrorCode::ControlStateUnavailable, + let revoke_request = RevokeDynamicCapabilityOriginOverlaysRequest { + overlay_ids: overlay_ids.into_iter().collect(), + }; + if site_id == app.controller_plan.site_id { + revoke_dynamic_capability_origin_overlays_local(app, &revoke_request).await?; + } else { + peer_controller_post_no_content_via_router( + app, &site_id, + "/v1/internal/dynamic-caps/origins/revoke", + &revoke_request, + ProtocolErrorCode::ControlStateUnavailable, "retract dynamic capability origin overlays", - format!("site router control endpoint is invalid: {err}"), ) - })?; - for overlay_id in overlay_ids { - revoke_route_overlay_with_retry(&endpoint, &overlay_id, Duration::from_secs(30)) - .await - .map_err(|err| { - controller_protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &site_id, - "retract dynamic capability origin overlays", - err, - ) - })?; + .await?; } } Ok(()) } -pub(super) fn child_site_publish_waves(child: &LiveChildRecord) -> Vec> { - let site_ids = child - .site_plans - .iter() - .map(|site_plan| site_plan.site_id.clone()) - .collect::>(); - let mut incoming = site_ids - .iter() - .map(|site_id| (site_id.clone(), BTreeSet::::new())) - .collect::>(); - let mut outgoing = site_ids - .iter() - .map(|site_id| (site_id.clone(), BTreeSet::::new())) - .collect::>(); - for link in child_link_records(child) { - if link.weak || link.provider_site == link.consumer_site { - continue; - } - if !site_ids.contains(&link.provider_site) || !site_ids.contains(&link.consumer_site) { - continue; - } - incoming - .get_mut(&link.consumer_site) - .expect("consumer site should be tracked") - .insert(link.provider_site.clone()); - outgoing - .get_mut(&link.provider_site) - .expect("provider site should be tracked") - .insert(link.consumer_site.clone()); - } - - let mut ready = incoming - .iter() - .filter(|(_, deps)| deps.is_empty()) - .map(|(site_id, _)| site_id.clone()) - .collect::>(); - let mut waves = Vec::new(); - let mut scheduled = BTreeSet::new(); - while !ready.is_empty() { - let wave = ready.iter().cloned().collect::>(); - let mut next_ready = BTreeSet::new(); - for site_id in &wave { - if !scheduled.insert(site_id.clone()) { - continue; - } - for consumer in outgoing - .get(site_id) - .into_iter() - .flat_map(|sites| sites.iter()) - { - let deps = incoming - .get_mut(consumer) - .expect("consumer dependencies should be tracked"); - deps.remove(site_id); - if deps.is_empty() { - next_ready.insert(consumer.clone()); - } - } - } - waves.push(wave); - ready = next_ready; - } - - let remaining = site_ids - .into_iter() - .filter(|site_id| scheduled.insert(site_id.clone())) - .collect::>(); - if !remaining.is_empty() { - waves.push(remaining); - } - waves -} - pub(super) fn cloned_child_record( state: &FrameworkControlState, child_id: u64, @@ -1560,10 +1831,10 @@ pub(super) fn cloned_child_record( pub(super) async fn rollback_prepared_sites( app: &ControlStateApp, child_id: u64, - prepared_sites: &[String], + prepared: bool, ) -> Result<()> { - for site_id in prepared_sites { - rollback_child_on_site(app, child_id, site_id).await?; + if prepared { + rollback_child_on_site(app, child_id, &app.controller_plan.site_id).await?; } Ok(()) } @@ -1595,123 +1866,41 @@ pub(super) async fn continue_create_committed_hidden( child }; - let site_plans = child - .site_plans - .iter() - .map(|site_plan| (site_plan.site_id.clone(), site_plan.clone())) - .collect::>(); - let child_sites = site_plans.keys().cloned().collect::>(); - let links = child_link_records(&child); - let mut published_child_sites = BTreeSet::new(); - let mut published_links = Vec::new(); - for wave in child_site_publish_waves(&child) { - for site_id in &wave { - for link in links.iter().filter(|link| link.consumer_site == *site_id) { - let provider_ready = !child_sites.contains(&link.provider_site) - || published_child_sites.contains(&link.provider_site); - if provider_ready { - publish_link_overlays(app, &child, link).await?; - published_links.push(link.clone()); - } + let state = app.control_state.lock().await.clone(); + let published_links = child_link_records(&child); + for link in &published_links { + publish_link_overlays(app, &child, link).await?; + } + if let Err(err) = publish_child_on_site_with_prepare_retry(app, &state, &child).await { + let mut cleanup_error = None; + for link in published_links.iter().rev() { + if let Err(retract_err) = retract_link_overlays(app, &child, link).await + && cleanup_error.is_none() + { + cleanup_error = Some(retract_err); } } - let mut publish_tasks = tokio::task::JoinSet::new(); - for site_id in &wave { - let app = app.clone(); - let child_id = child.child_id; - let site_plan = site_plans - .get(site_id) - .expect("site plan should exist for wave site") - .clone(); - let site_id = site_id.clone(); - publish_tasks.spawn(async move { - publish_child_on_site_with_prepare_retry(&app, child_id, &site_plan) - .await - .map(|_| site_id) - }); - } - let mut first_error = None; - let mut published_wave_sites = Vec::new(); - while let Some(result) = publish_tasks.join_next().await { - match result { - Ok(Ok(site_id)) => published_wave_sites.push(site_id), - Ok(Err(err)) if first_error.is_none() => first_error = Some(err), - Ok(Err(_)) => {} - Err(err) if first_error.is_none() => { - first_error = Some(protocol_error( - ProtocolErrorCode::PublishFailed, - &format!("site publish task failed: {err}"), - )); - } - Err(_) => {} - } + if let Err(destroy_err) = destroy_child_on_site(app, &state, &child).await + && cleanup_error.is_none() + { + cleanup_error = Some(destroy_err); } - if let Some(err) = first_error { - let desired_site_plans = { - let state = app.control_state.lock().await; - desired_site_plan_map(&state, &child_sites)? - }; - let mut cleanup_error = None; - for link in published_links.iter().rev() { - if let Err(retract_err) = retract_link_overlays(app, &child, link).await - && cleanup_error.is_none() - { - cleanup_error = Some(retract_err); - } - } - - let mut destroy_tasks = tokio::task::JoinSet::new(); - for site_id in &child_sites { - let app = app.clone(); - let desired_site_plan = desired_site_plans.get(site_id).cloned(); - let site_id = site_id.clone(); - destroy_tasks.spawn(async move { - destroy_child_on_site(&app, child.child_id, &site_id, desired_site_plan).await - }); - } - while let Some(result) = destroy_tasks.join_next().await { - match result { - Ok(Ok(())) => {} - Ok(Err(destroy_err)) if cleanup_error.is_none() => { - cleanup_error = Some(destroy_err); - } - Ok(Err(_)) => {} - Err(join_err) if cleanup_error.is_none() => { - cleanup_error = Some(protocol_error( - ProtocolErrorCode::PublishFailed, - &format!("site destroy task failed: {join_err}"), - )); - } - Err(_) => {} - } - } - - if let Some(cleanup_error) = cleanup_error { - return Err(protocol_error( - ProtocolErrorCode::PublishFailed, - &format!("{}; cleanup failed: {}", err.message, cleanup_error.message), - )); - } - - let mut state = app.control_state.lock().await; - if child_record_location(&state, child.child_id).is_ok() { - let tx_id = child_create_tx_id(&state, child.child_id)?; - persist_control_state_update( - &mut state, - &app.state_path, - "create_aborted", - |state| { - append_journal_entry(state, tx_id, &child, ChildState::CreateAborted); - remove_child_record(state, child.child_id)?; - Ok(()) - }, - )?; - } - return Err(err); + if let Some(cleanup_error) = cleanup_error { + return Err(protocol_error( + ProtocolErrorCode::PublishFailed, + &format!("{}; cleanup failed: {}", err.message, cleanup_error.message), + )); } - for site_id in published_wave_sites { - published_child_sites.insert(site_id); + let mut state = app.control_state.lock().await; + if child_record_location(&state, child.child_id).is_ok() { + let tx_id = child_create_tx_id(&state, child.child_id)?; + persist_control_state_update(&mut state, &app.state_path, "create_aborted", |state| { + append_journal_entry(state, tx_id, &child, ChildState::CreateAborted); + remove_child_record(state, child.child_id)?; + Ok(()) + })?; } + return Err(err); } publish_child_overlays(app, &child).await?; @@ -1743,43 +1932,8 @@ pub(super) async fn continue_destroy_retracted( } child }; - let desired_site_plans = { - let state = app.control_state.lock().await; - desired_site_plan_map( - &state, - &child - .site_plans - .iter() - .map(|site_plan| site_plan.site_id.clone()) - .collect(), - )? - }; - let mut destroy_tasks = tokio::task::JoinSet::new(); - for site_plan in child.site_plans.clone() { - let app = app.clone(); - let desired_site_plan = desired_site_plans.get(&site_plan.site_id).cloned(); - destroy_tasks.spawn(async move { - destroy_child_on_site(&app, child.child_id, &site_plan.site_id, desired_site_plan).await - }); - } - let mut first_error = None; - while let Some(result) = destroy_tasks.join_next().await { - match result { - Ok(Ok(())) => {} - Ok(Err(err)) if first_error.is_none() => first_error = Some(err), - Ok(Err(_)) => {} - Err(err) if first_error.is_none() => { - first_error = Some(protocol_error( - ProtocolErrorCode::PublishFailed, - &format!("site destroy task failed: {err}"), - )); - } - Err(_) => {} - } - } - if let Some(err) = first_error { - return Err(err); - } + let state = app.control_state.lock().await.clone(); + destroy_child_on_site(app, &state, &child).await?; let mut state = app.control_state.lock().await; let child = cloned_child_record(&state, child_id)?; @@ -1849,11 +2003,12 @@ pub(super) async fn execute_create_child( (tx_id, child) }; let (tx_id, child) = child; + reconcile_local_framework_routes(app).await?; - let mut prepared_sites = Vec::new(); - for site_plan in &child.site_plans { - if let Err(err) = prepare_child_on_site(app, child.child_id, site_plan).await { - let rollback_err = rollback_prepared_sites(app, child.child_id, &prepared_sites).await; + let state = app.control_state.lock().await.clone(); + if let Err(err) = prepare_child_on_site(app, &state, &child).await { + let rollback_err = rollback_prepared_sites(app, child.child_id, false).await; + let should_reconcile = { let mut state = app.control_state.lock().await; if state .pending_creates @@ -1871,18 +2026,23 @@ pub(super) async fn execute_create_child( Ok(()) }, )?; - } - let err = if let Err(rollback_err) = rollback_err { - protocol_error( - ProtocolErrorCode::PrepareFailed, - &format!("{}; rollback failed: {rollback_err}", err.message), - ) + true } else { - err - }; - return Err(err.into()); + false + } + }; + if should_reconcile { + reconcile_local_framework_routes(app).await?; } - prepared_sites.push(site_plan.site_id.clone()); + let err = if let Err(rollback_err) = rollback_err { + protocol_error( + ProtocolErrorCode::PrepareFailed, + &format!("{}; rollback failed: {rollback_err}", err.message), + ) + } else { + err + }; + return Err(err.into()); } { @@ -1927,7 +2087,7 @@ pub(super) async fn execute_destroy_child( child_name: &str, ) -> std::result::Result<(), ProtocolApiError> { let _authority_guard = acquire_authority_lock(app, authority_realm_id).await; - let next = { + let (next, reconcile_routes) = { let mut state = app.control_state.lock().await; let Some(child) = state .live_children @@ -1953,10 +2113,10 @@ pub(super) async fn execute_destroy_child( Ok(()) }, )?; - (child.child_id, ChildState::DestroyRequested) + ((child.child_id, ChildState::DestroyRequested), true) } - ChildState::DestroyRequested => (child.child_id, ChildState::DestroyRequested), - ChildState::DestroyRetracted => (child.child_id, ChildState::DestroyRetracted), + ChildState::DestroyRequested => ((child.child_id, ChildState::DestroyRequested), false), + ChildState::DestroyRetracted => ((child.child_id, ChildState::DestroyRetracted), false), ChildState::DestroyCommitted | ChildState::CreateAborted => return Ok(()), _ => { return Err(protocol_error( @@ -1967,6 +2127,9 @@ pub(super) async fn execute_destroy_child( } } }; + if reconcile_routes { + reconcile_local_framework_routes(app).await?; + } match next.1 { ChildState::DestroyRequested => continue_destroy_requested(app, next.0).await?, ChildState::DestroyRetracted => continue_destroy_retracted(app, next.0).await?, @@ -2001,12 +2164,7 @@ pub(super) async fn recover_control_state(app: &ControlStateApp) -> Result<()> { } } ChildState::CreatePrepared => { - let prepared_sites = child - .site_plans - .iter() - .map(|site_plan| site_plan.site_id.clone()) - .collect::>(); - rollback_prepared_sites(app, child.child_id, &prepared_sites) + rollback_prepared_sites(app, child.child_id, true) .await .wrap_err_with(|| { format!( @@ -2053,5 +2211,8 @@ pub(super) async fn recover_control_state(app: &ControlStateApp) -> Result<()> { ChildState::CreateAborted | ChildState::DestroyCommitted => {} } } + reconcile_local_framework_routes(app) + .await + .map_err(|err| miette::miette!(err.message))?; Ok(()) } diff --git a/runtime/site-controller/src/planner.rs b/runtime/site-controller/src/planner.rs index f7787472..a4061afd 100644 --- a/runtime/site-controller/src/planner.rs +++ b/runtime/site-controller/src/planner.rs @@ -1,4 +1,16 @@ use super::{api::*, state::*, *}; +use crate::runtime_api::SharedSiteControllerRuntime; + +#[derive(Clone, Debug)] +pub(crate) struct LocalChildRuntimeSpec { + pub(crate) component_ids: Vec, + pub(crate) assigned_components: Vec, + pub(crate) child_artifact_files: BTreeMap, + pub(crate) site_artifact_files: BTreeMap, + pub(crate) proxy_exports: BTreeMap, + pub(crate) direct_inputs: Vec, + pub(crate) routed_inputs: Vec, +} pub(super) async fn prepare_child_record( state: &mut FrameworkControlState, @@ -76,7 +88,6 @@ pub(super) async fn prepare_child_record( fragment: Some(fragment), input_bindings: child_input_binding_records(&live_scenario, &resolved_bindings), assignments: BTreeMap::new(), - site_plans: Vec::new(), overlay_ids: Vec::new(), overlays: Vec::new(), outputs, @@ -126,18 +137,27 @@ pub(super) fn rebuild_live_child_runtime_metadata( &format!("child placement could not be satisfied: {err}"), ) })?; + let fragment_sites = fragment + .components + .iter() + .filter_map(|component| planned.assignments.get(component.moniker.as_str())) + .cloned() + .collect::>(); + if fragment_sites.len() > 1 { + return Err(protocol_error( + ProtocolErrorCode::PlacementUnsatisfied, + &format!( + "dynamic child `{}` spans multiple sites ({}) but site controllers only create \ + children within a single site", + child.name, + fragment_sites.into_iter().collect::>().join(", ") + ), + )); + } child.assignments = planned.assignments; child.overlays = dynamic_overlay_records(&planned.incident_links, fragment); let mut live_assignments = existing_assignments.clone(); live_assignments.extend(child.assignments.clone()); - child.site_plans = dynamic_site_plans( - &planned.site_plans, - &child.assignments, - fragment, - &child.outputs, - &child.overlays, - &dynamic_input_route_records(&live_assignments, fragment, &child.input_bindings), - )?; child.overlay_ids = child .overlays .iter() @@ -1300,14 +1320,20 @@ pub(super) fn remove_child_record( Ok(()) } -pub(super) fn dynamic_site_plans( - desired_site_plans: &BTreeMap, +struct LocalChildRuntimeSpecInputs<'a> { + outputs: &'a BTreeMap, + overlays: &'a [DynamicOverlayRecord], + direct_inputs: &'a [DynamicInputDirectRecord], + routed_inputs: &'a [DynamicInputRouteRecord], +} + +fn build_local_child_runtime_spec_from_site_plan( + site_id: &str, + desired_site_plan: &amber_compiler::run_plan::RunSitePlan, assignments: &BTreeMap, fragment: &LiveScenarioFragment, - outputs: &BTreeMap, - overlays: &[DynamicOverlayRecord], - routed_inputs: &[DynamicInputRouteRecord], -) -> std::result::Result, ProtocolErrorResponse> { + inputs: LocalChildRuntimeSpecInputs<'_>, +) -> std::result::Result { let child_moniker_set = fragment .components .iter() @@ -1318,145 +1344,293 @@ pub(super) fn dynamic_site_plans( .iter() .map(|component| (component.id, component.moniker.as_str())) .collect::>(); - let mut site_plans = Vec::new(); - for (site_id, desired_site_plan) in desired_site_plans { - let component_ids = desired_site_plan - .scenario_ir - .components - .iter() - .filter(|component| child_moniker_set.contains(component.moniker.as_str())) - .map(|component| component.id) - .collect::>(); - if component_ids.is_empty() { - continue; - } - let assigned_components = desired_site_plan - .assigned_components - .iter() - .filter(|moniker| { - fragment - .components - .iter() - .any(|component| component.moniker == **moniker) - }) - .cloned() - .collect::>(); - let mut proxy_exports = BTreeMap::new(); - for (name, output) in outputs { - let Some((component_id, component_moniker, provide_name)) = - output.sources.iter().find_map(|source| { - let BindingFromIr::Component { component, provide } = &source.from else { - return None; - }; - let moniker = child_monikers.get(component)?; - (assignments.get(*moniker)? == site_id).then_some(( - *component, - *moniker, - provide.as_str(), - )) - }) - else { - continue; - }; - let protocol = match output.decl.kind.transport() { - CapabilityTransport::Http => "http", - CapabilityTransport::NonNetwork => continue, - _ => continue, - }; - let export = - dynamic_proxy_export_record(fragment, component_id, provide_name, protocol) - .ok_or_else(|| { - protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &format!( - "dynamic export `{name}` on component `{component_moniker}` could \ - not be resolved to a concrete network endpoint" - ), - ) - })?; - proxy_exports.insert(name.clone(), export); - } - for overlay in overlays { - let DynamicOverlayAction::ExportPeer { link } = &overlay.action else { - continue; - }; - if link.provider_site != *site_id - || !child_moniker_set.contains(link.provider_component.as_str()) - { - continue; - } - if proxy_exports.contains_key(&link.export_name) { - continue; - } - let component = fragment + let component_ids = desired_site_plan + .scenario_ir + .components + .iter() + .filter(|component| child_moniker_set.contains(component.moniker.as_str())) + .map(|component| component.id) + .collect::>(); + if component_ids.is_empty() { + return Err(protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("site `{site_id}` has no child-owned components in the desired site plan"), + )); + } + let assigned_components = desired_site_plan + .assigned_components + .iter() + .filter(|moniker| { + fragment .components .iter() - .find(|component| component.moniker == link.provider_component) - .ok_or_else(|| { - protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &format!( - "dynamic export provider `{}` is missing from the live child fragment", - link.provider_component - ), - ) - })?; - let export = dynamic_proxy_export_record( - fragment, - component.id, - &link.provide, - &link.protocol.to_string(), - ) + .any(|component| component.moniker == **moniker) + }) + .cloned() + .collect::>(); + let mut proxy_exports = BTreeMap::new(); + for (name, output) in inputs.outputs { + let Some((component_id, component_moniker, provide_name)) = + output.sources.iter().find_map(|source| { + let BindingFromIr::Component { component, provide } = &source.from else { + return None; + }; + let moniker = child_monikers.get(component)?; + (assignments.get(*moniker)? == site_id).then_some(( + *component, + *moniker, + provide.as_str(), + )) + }) + else { + continue; + }; + let protocol = match output.decl.kind.transport() { + CapabilityTransport::Http => "http", + CapabilityTransport::NonNetwork => continue, + _ => continue, + }; + let export = dynamic_proxy_export_record(fragment, component_id, provide_name, protocol) .ok_or_else(|| { protocol_error( ProtocolErrorCode::ControlStateUnavailable, &format!( - "dynamic export `{}` on component `{}` could not be resolved to a \ - concrete network endpoint", - link.export_name, link.provider_component + "dynamic export `{name}` on component `{component_moniker}` could not be \ + resolved to a concrete network endpoint" ), ) })?; - proxy_exports.insert(link.export_name.clone(), export); + proxy_exports.insert(name.clone(), export); + } + for overlay in inputs.overlays { + let DynamicOverlayAction::ExportPeer { link } = &overlay.action else { + continue; + }; + if link.provider_site != site_id + || !child_moniker_set.contains(link.provider_component.as_str()) + { + continue; } - let artifact_files = if desired_site_plan.site.kind == SiteKind::Kubernetes { - project_kubernetes_dynamic_child_artifact_files( - &desired_site_plan.artifact_files, - &component_ids, - ) - .map_err(|err| { + if proxy_exports.contains_key(&link.export_name) { + continue; + } + let component = fragment + .components + .iter() + .find(|component| component.moniker == link.provider_component) + .ok_or_else(|| { protocol_error( ProtocolErrorCode::ControlStateUnavailable, &format!( - "failed to project kubernetes child artifact for site `{site_id}`: {err}" + "dynamic export provider `{}` is missing from the live child fragment", + link.provider_component ), ) - })? - } else { - desired_site_plan.artifact_files.clone() - }; - site_plans.push(DynamicSitePlanRecord { - site_id: site_id.clone(), - kind: desired_site_plan.site.kind, - router_identity_id: desired_site_plan.router_identity_id.clone(), - component_ids, - assigned_components, - artifact_files, - desired_artifact_files: desired_site_plan.artifact_files.clone(), - proxy_exports, - routed_inputs: routed_inputs - .iter() - .filter(|input| input.component == child_monikers[&fragment.root_component_id]) - .filter(|input| { - assignments - .get(input.component.as_str()) - .is_some_and(|assigned_site| assigned_site == site_id) - }) - .cloned() - .collect(), - }); + })?; + let export = dynamic_proxy_export_record( + fragment, + component.id, + &link.provide, + &link.protocol.to_string(), + ) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "dynamic export `{}` on component `{}` could not be resolved to a concrete \ + network endpoint", + link.export_name, link.provider_component + ), + ) + })?; + proxy_exports.insert(link.export_name.clone(), export); } - Ok(site_plans) + let child_artifact_files = if desired_site_plan.site.kind == SiteKind::Kubernetes { + project_kubernetes_dynamic_child_artifact_files( + &desired_site_plan.artifact_files, + &component_ids, + ) + .map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("failed to project kubernetes child artifact for site `{site_id}`: {err}"), + ) + })? + } else { + desired_site_plan.artifact_files.clone() + }; + Ok(LocalChildRuntimeSpec { + component_ids, + assigned_components, + child_artifact_files, + site_artifact_files: desired_site_plan.artifact_files.clone(), + proxy_exports, + direct_inputs: inputs + .direct_inputs + .iter() + .filter(|input| input.component == child_monikers[&fragment.root_component_id]) + .filter(|input| { + assignments + .get(input.component.as_str()) + .is_some_and(|assigned_site| assigned_site == site_id) + }) + .cloned() + .collect(), + routed_inputs: inputs + .routed_inputs + .iter() + .filter(|input| input.component == child_monikers[&fragment.root_component_id]) + .filter(|input| { + assignments + .get(input.component.as_str()) + .is_some_and(|assigned_site| assigned_site == site_id) + }) + .cloned() + .collect(), + }) +} + +pub(super) fn child_runtime_site_id( + child: &LiveChildRecord, +) -> std::result::Result { + let fragment = child.fragment.as_ref().ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "dynamic child `{}` is missing the authoritative fragment required for placement", + child.name + ), + ) + })?; + let site_ids = fragment + .components + .iter() + .filter_map(|component| child.assignments.get(component.moniker.as_str())) + .cloned() + .collect::>(); + match site_ids.len() { + 1 => Ok(site_ids + .into_iter() + .next() + .expect("single child site id should be present")), + 0 => Err(protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "dynamic child `{}` has no site assignment for any fragment component", + child.name + ), + )), + _ => Err(protocol_error( + ProtocolErrorCode::PlacementUnsatisfied, + &format!( + "dynamic child `{}` spans multiple sites ({}) but site controllers only create \ + children within a single site", + child.name, + site_ids.into_iter().collect::>().join(", ") + ), + )), + } +} + +pub(super) fn live_assignment_map_with_child( + state: &FrameworkControlState, + child: &LiveChildRecord, +) -> BTreeMap { + let mut assignments = live_assignment_map(state); + assignments.extend(child.assignments.clone()); + assignments +} + +pub(super) fn build_local_child_runtime_spec( + state: &FrameworkControlState, + child: &LiveChildRecord, + site_id: &str, +) -> std::result::Result { + let fragment = child.fragment.as_ref().ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "dynamic child `{}` is missing the authoritative fragment required for runtime \ + preparation", + child.name + ), + ) + })?; + let scenario = scenario_with_fragment(&live_scenario_ir(state)?, fragment)?; + let live_assignments = live_assignment_map_with_child(state, child); + let desired_site_plan = build_site_plan_subset( + &scenario, + &placement_file_from_state(state), + &run_plan_activation_from_state(state), + &live_assignments, + &BTreeSet::from([site_id.to_string()]), + ) + .map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "failed to build desired site plan for local child `{}` on site `{site_id}`: {err}", + child.name + ), + ) + })? + .remove(site_id) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "desired site plan for local child `{}` is missing site `{site_id}`", + child.name + ), + ) + })?; + build_local_child_runtime_spec_from_site_plan( + site_id, + &desired_site_plan, + &live_assignments, + fragment, + LocalChildRuntimeSpecInputs { + outputs: &child.outputs, + overlays: &child.overlays, + direct_inputs: &dynamic_input_direct_records( + &live_assignments, + fragment, + &child.input_bindings, + ), + routed_inputs: &dynamic_input_route_records( + &live_assignments, + fragment, + &child.input_bindings, + ), + }, + ) +} + +pub(super) fn build_desired_site_artifact_files( + state: &FrameworkControlState, + site_id: &str, +) -> std::result::Result, ProtocolErrorResponse> { + build_site_plan_subset( + &decode_live_scenario(state)?, + &placement_file_from_state(state), + &run_plan_activation_from_state(state), + &live_assignment_map(state), + &BTreeSet::from([site_id.to_string()]), + ) + .map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("failed to build desired site artifacts for site `{site_id}`: {err}"), + ) + })? + .remove(site_id) + .map(|site_plan| site_plan.artifact_files) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("desired site artifact snapshot is missing site `{site_id}`"), + ) + }) } pub(super) fn child_input_binding_records( @@ -1525,7 +1699,61 @@ pub(super) fn dynamic_input_route_records( CapabilityTransport::NonNetwork => return None, _ => return None, }; - (provider_site == child_site).then(|| DynamicInputRouteRecord { + (provider_site != child_site).then(|| DynamicInputRouteRecord { + component: root_component.moniker.clone(), + slot: binding.slot.clone(), + provider_component: provider_component.clone(), + protocol: protocol.to_string(), + capability_kind: binding.decl.kind.to_string(), + capability_profile: binding.decl.profile.clone(), + target: DynamicInputRouteTarget::ComponentProvide { + provide: provide.clone(), + }, + }) + }) + .collect() +} + +pub(super) fn dynamic_input_direct_records( + assignments: &BTreeMap, + fragment: &LiveScenarioFragment, + input_bindings: &[ChildInputBindingRecord], +) -> Vec { + let Some(root_component) = fragment + .components + .iter() + .find(|component| component.id == fragment.root_component_id) + else { + return Vec::new(); + }; + let Some(child_site) = assignments.get(root_component.moniker.as_str()) else { + return Vec::new(); + }; + let fragment_components = fragment + .components + .iter() + .map(|component| component.moniker.as_str()) + .collect::>(); + input_bindings + .iter() + .filter_map(|binding| { + let [source] = binding.sources.as_slice() else { + return None; + }; + let BindingFromIr::Component { provide, .. } = &source.from else { + return None; + }; + let provider_component = source.component_moniker.as_ref()?; + if fragment_components.contains(provider_component.as_str()) { + return None; + } + let provider_site = assignments.get(provider_component.as_str())?; + let protocol = match binding.decl.kind.transport() { + CapabilityTransport::Http => "http", + CapabilityTransport::NonNetwork => return None, + _ => return None, + }; + (provider_site == child_site).then(|| DynamicInputDirectRecord { component: root_component.moniker.clone(), slot: binding.slot.clone(), provider_component: provider_component.clone(), @@ -3049,7 +3277,6 @@ pub(super) struct ControlStateApp { pub(super) mesh_scope: Arc, pub(super) control_state_auth_token: Arc, pub(super) controller_plan: Arc, - pub(super) peer_controllers: Arc>, pub(super) authority_locks: Arc>>>>, pub(super) runtime: SharedSiteControllerRuntime, } diff --git a/runtime/site-controller/src/runtime_api.rs b/runtime/site-controller/src/runtime_api.rs index e796ec02..f2d7b982 100644 --- a/runtime/site-controller/src/runtime_api.rs +++ b/runtime/site-controller/src/runtime_api.rs @@ -8,13 +8,13 @@ use std::{ }; use amber_compiler::run_plan::{RunLink, SiteKind}; -use amber_mesh::{MeshConfigPublic, MeshIdentityPublic}; +use amber_mesh::{InboundRoute, MeshConfigPublic, MeshIdentityPublic, MeshPeer}; use amber_proxy::ControlEndpoint; use base64::Engine as _; use miette::{IntoDiagnostic as _, Result, WrapErr as _}; use serde::{Deserialize, Serialize}; -use super::state::{DynamicSitePlanRecord, SiteControllerPlan}; +use super::state::{FrameworkControlState, LiveChildRecord, SiteControllerPlan}; const KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH: &str = "01-configmaps/amber-mesh-provision.yaml"; const KUBERNETES_PROVISIONER_JOB_PATH: &str = "02-rbac/amber-provisioner-job.yaml"; @@ -27,7 +27,7 @@ const SITE_CONTROLLER_RUNTIME_PLAN_VERSION: u32 = 1; pub type SiteControllerRuntimeFuture<'a, T> = Pin> + Send + 'a>>; -pub trait SiteControllerRuntime: Send + Sync { +pub(crate) trait SiteControllerRuntime: Send + Sync { fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()>; fn resolve_link_external_url<'a>( @@ -42,15 +42,15 @@ pub trait SiteControllerRuntime: Send + Sync { fn prepare_child<'a>( &'a self, plan: &'a SiteControllerPlan, - child_id: u64, - site_plan: DynamicSitePlanRecord, + state: FrameworkControlState, + child: LiveChildRecord, ) -> SiteControllerRuntimeFuture<'a, ()>; fn publish_child<'a>( &'a self, plan: &'a SiteControllerPlan, - child_id: u64, - site_plan: DynamicSitePlanRecord, + state: FrameworkControlState, + child: LiveChildRecord, ) -> SiteControllerRuntimeFuture<'a, ()>; fn rollback_child<'a>( @@ -62,8 +62,8 @@ pub trait SiteControllerRuntime: Send + Sync { fn destroy_child<'a>( &'a self, plan: &'a SiteControllerPlan, - child_id: u64, - desired_site_plan: Option, + state: FrameworkControlState, + child: LiveChildRecord, ) -> SiteControllerRuntimeFuture<'a, ()>; fn collect_live_component_runtime_metadata( @@ -110,7 +110,7 @@ pub trait SiteControllerRuntime: Send + Sync { ) -> Result<()>; } -pub type SharedSiteControllerRuntime = Arc; +pub(crate) type SharedSiteControllerRuntime = Arc; #[derive(Clone, Debug, Serialize, Deserialize)] pub struct SiteReceipt { @@ -151,6 +151,8 @@ pub struct SiteControllerRuntimePlan { pub site_id: String, pub kind: SiteKind, pub router_identity_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub local_router_control: Option, pub artifact_dir: String, pub site_state_root: String, pub listen_addr: SocketAddr, @@ -176,6 +178,7 @@ pub struct SiteControllerRuntimePlan { pub struct LiveComponentRuntimeMetadata { pub moniker: String, pub host_mesh_addr: String, + pub control_endpoint: Option, pub mesh_config: MeshConfigPublic, } @@ -195,6 +198,14 @@ pub struct DesiredExportPeerOverlay { pub route_id: Option, } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct DesiredRouteOverlay { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub peers: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub inbound_routes: Vec, +} + #[derive(Clone, Debug)] pub struct LaunchedSite { pub receipt: SiteReceipt, @@ -215,6 +226,7 @@ pub fn site_controller_runtime_plan_from_controller_plan( site_id: plan.site_id.clone(), kind: plan.kind, router_identity_id: plan.router_identity_id.clone(), + local_router_control: plan.local_router_control.clone(), artifact_dir: plan.artifact_dir.clone(), site_state_root: plan.site_state_root.clone(), listen_addr: plan.listen_addr, diff --git a/runtime/site-controller/src/site_controller.rs b/runtime/site-controller/src/site_controller.rs index 7a338083..cf8dd0e0 100644 --- a/runtime/site-controller/src/site_controller.rs +++ b/runtime/site-controller/src/site_controller.rs @@ -1,12 +1,14 @@ use std::sync::Arc; -use amber_mesh::dynamic_caps::HeldListResponse; +use amber_mesh::{MeshIdentityPublic, dynamic_caps::HeldListResponse}; +use amber_proxy::{ControlEndpoint, fetch_router_identity}; use axum::{ Json, Router, extract::{Path as AxumPath, State}, - http::{HeaderMap, Method, StatusCode}, + http::{HeaderMap, StatusCode}, routing::{get, post}, }; +use base64::Engine as _; use miette::{IntoDiagnostic as _, Result, WrapErr as _}; use reqwest::Client as ReqwestClient; use serde::{Serialize, de::DeserializeOwned}; @@ -16,22 +18,28 @@ use super::{ ccs_api::{self, FrameworkComponentInspectRequest, FrameworkComponentInspectResponse}, control_state_api::{ self, DynamicCapsInspectRequest, DynamicCapsInspectResponse, DynamicCapsMutateRequest, - DynamicCapsMutateResponse, + DynamicCapsMutateResponse, local_component_runtime, + resolve_dynamic_capability_origin_internal, }, dynamic_caps::{ self, ControlDynamicHeldDetailRequest, ControlDynamicHeldListRequest, ControlDynamicInspectRefRequest, ControlDynamicResolveOriginRequest, ControlDynamicRevokeRequest, ControlDynamicShareRequest, + InternalDynamicResolveOriginRequest, }, http::{ authorize_framework_auth_header, cleanup_dynamic_bridge_proxies, healthz, read_json, required_header, shutdown_signal, }, orchestration::{ - ControlCreateChildRequest, ControlDestroyChildRequest, ProtocolApiError, - SiteControllerDestroyRequest, SiteControllerPrepareRequest, SiteControllerPublishRequest, - execute_create_child, execute_destroy_child, publish_dynamic_capability_origin_local, - recover_control_state, + ClearExportPeerOverlayRequest, ClearExternalSlotOverlayRequest, ProtocolApiError, + PublishExportPeerOverlayRequest, PublishExternalSlotOverlayRequest, + ResolveExternalLinkUrlRequest, ResolveExternalLinkUrlResponse, + RevokeDynamicCapabilityOriginOverlaysRequest, clear_export_peer_overlay_local, + clear_external_slot_overlay_local, execute_create_child, execute_destroy_child, + publish_dynamic_capability_origin_local, publish_export_peer_overlay_local, + publish_external_slot_overlay_local, recover_control_state, + resolve_external_link_url_local, revoke_dynamic_capability_origin_overlays_local, }, planner::{ ControlStateApp, LocalDynamicCapabilityOriginApp, SiteControllerApp, protocol_error, @@ -39,11 +47,11 @@ use super::{ state::*, *, }; -use crate::api::capability_instance_record; +use crate::runtime_api::SharedSiteControllerRuntime; -const CONTROLLER_LOCAL_ONLY_HEADER: &str = "x-amber-site-controller-local-only"; +pub(crate) const CONTROLLER_LOCAL_ONLY_HEADER: &str = "x-amber-site-controller-local-only"; -pub async fn run_site_controller( +pub(crate) async fn run_site_controller( plan_path: PathBuf, runtime: SharedSiteControllerRuntime, ) -> Result<()> { @@ -60,7 +68,6 @@ pub async fn run_site_controller( mesh_scope: Arc::::from(plan.mesh_scope.clone()), control_state_auth_token: Arc::::from(plan.auth_token.clone()), controller_plan: Arc::new(plan.clone()), - peer_controllers: Arc::new(plan.peer_controllers.clone()), authority_locks: Arc::new(Mutex::new(BTreeMap::new())), runtime, }; @@ -102,6 +109,10 @@ pub(crate) fn site_controller_router(app_state: SiteControllerApp) -> Router { SITE_CONTROLLER_STATE_PATH, get(get_site_controller_state_route), ) + .route( + "/v1/controller/router-identity", + get(get_router_identity_route), + ) .route("/v1/templates", get(list_templates)) .route("/v1/templates/{template}", get(describe_template)) .route("/v1/templates/{template}/resolve", post(resolve_template)) @@ -115,10 +126,33 @@ pub(crate) fn site_controller_router(app_state: SiteControllerApp) -> Router { "/v1/internal/dynamic-caps/origins/publish", post(publish_dynamic_origin), ) - .route("/v1/controller/children", post(control_create_child_route)) .route( - "/v1/controller/children/{child}/destroy", - post(control_destroy_child_route), + "/v1/internal/dynamic-caps/resolve-origin", + post(resolve_dynamic_origin_internal_route), + ) + .route( + "/v1/internal/dynamic-caps/origins/revoke", + post(revoke_dynamic_origin_overlays), + ) + .route( + "/v1/internal/link-overlays/external-url", + post(resolve_external_link_url), + ) + .route( + "/v1/internal/link-overlays/external-slot/publish", + post(publish_external_slot_overlay_route), + ) + .route( + "/v1/internal/link-overlays/external-slot/clear", + post(clear_external_slot_overlay_route), + ) + .route( + "/v1/internal/link-overlays/export-peer/publish", + post(publish_export_peer_overlay_route), + ) + .route( + "/v1/internal/link-overlays/export-peer/clear", + post(clear_export_peer_overlay_route), ) .route( "/v1/controller/dynamic-caps/held", @@ -144,22 +178,6 @@ pub(crate) fn site_controller_router(app_state: SiteControllerApp) -> Router { "/v1/controller/dynamic-caps/resolve-origin", post(control_dynamic_resolve_origin_route), ) - .route( - "/v1/controller/site/children/{child_id}/prepare", - post(site_prepare_child_route), - ) - .route( - "/v1/controller/site/children/{child_id}/publish", - post(site_publish_child_route), - ) - .route( - "/v1/controller/site/children/{child_id}/rollback", - post(site_rollback_child_route), - ) - .route( - "/v1/controller/site/children/{child_id}/destroy", - post(site_destroy_child_route), - ) .with_state(app_state) } @@ -171,6 +189,74 @@ async fn get_site_controller_state_route( Ok(Json(app.control.control_state.lock().await.clone())) } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct RouterIdentityResponse { + pub(crate) id: String, + pub(crate) public_key_b64: String, +} + +fn authorize_local_controller_request( + app: &SiteControllerApp, + headers: &HeaderMap, +) -> std::result::Result<(), ProtocolApiError> { + authorize_framework_auth_header(headers, app.router_auth_token.as_ref())?; + if !controller_local_only(headers) { + return Err(ProtocolApiError::unauthorized( + "local controller endpoint requires router-local forwarding".to_string(), + )); + } + Ok(()) +} + +fn local_router_control_endpoint( + app: &SiteControllerApp, +) -> std::result::Result { + if let Some(raw) = app.control.controller_plan.local_router_control.as_deref() { + return parse_control_endpoint(raw).map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "site controller local router control endpoint is invalid: {err}" + )) + }); + } + let manager_state_path = + Path::new(&app.control.controller_plan.site_state_root).join("manager-state.json"); + if manager_state_path.is_file() { + let state: super::planner::SiteManagerStateView = + read_json(&manager_state_path, "site manager state").map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "site manager state is unavailable: {err}" + )) + })?; + if let Some(raw) = state.router_control { + return parse_control_endpoint(&raw).map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "site manager state has an invalid router control endpoint: {err}" + )) + }); + } + } + Err(ProtocolApiError::control_state_unavailable( + "site controller local router control endpoint is unavailable".to_string(), + )) +} + +async fn get_router_identity_route( + State(app): State, + headers: HeaderMap, +) -> std::result::Result, ProtocolApiError> { + authorize_local_controller_request(&app, &headers)?; + let endpoint = local_router_control_endpoint(&app)?; + let identity = fetch_router_identity(&endpoint).await.map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "failed to read local router identity: {err}" + )) + })?; + Ok(Json(RouterIdentityResponse { + id: identity.id, + public_key_b64: base64::engine::general_purpose::STANDARD.encode(identity.public_key), + })) +} + pub(super) async fn authorize_public_request( app: &SiteControllerApp, headers: &HeaderMap, @@ -192,59 +278,60 @@ fn controller_local_only(headers: &HeaderMap) -> bool { .is_some_and(|value| value == "1") } -fn peer_controller_url<'a>( - app: &'a SiteControllerApp, +fn peer_site_router_url_for_site( + app: &SiteControllerApp, site_id: &str, -) -> std::result::Result<&'a str, ProtocolApiError> { - if site_id == app.control.controller_plan.site_id { - return Ok(app.control.controller_plan.authority_url.as_str()); - } +) -> std::result::Result { app.control - .peer_controllers + .controller_plan + .peer_site_router_urls .get(site_id) - .map(|plan| plan.authority_url.as_str()) + .cloned() .ok_or_else(|| { ProtocolApiError::control_state_unavailable(format!( - "site `{site_id}` controller metadata is unavailable" + "site controller `{}` has no router forward path to peer site `{site_id}`", + app.control.controller_plan.site_id )) }) } -async fn forward_framework_request( +fn peer_site_ids(app: &SiteControllerApp) -> Vec { + app.control + .controller_plan + .peer_site_router_urls + .keys() + .cloned() + .collect() +} + +async fn peer_dynamic_caps_post_via_router( app: &SiteControllerApp, site_id: &str, - method: Method, path: &str, - record: &CapabilityInstanceRecord, - body: Option<&TReq>, + body: &TReq, ) -> std::result::Result { - let mut request = app + let response = app .control .client - .request( - reqwest::Method::from_bytes(method.as_str().as_bytes()) - .expect("axum HTTP method should map to reqwest"), - format!( - "{}{}", - peer_controller_url(app, site_id)?.trim_end_matches('/'), - path - ), - ) - .header(FRAMEWORK_AUTH_HEADER, app.router_auth_token.as_ref()) - .header(FRAMEWORK_ROUTE_ID_HEADER, record.route_id.as_str()) - .header(FRAMEWORK_PEER_ID_HEADER, record.recipient_peer_id.as_str()); - if let Some(body) = body { - request = request.json(body); - } - let response = request.send().await.map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "failed to reach site controller for site `{site_id}`: {err}" + .post(format!( + "{}{}", + peer_site_router_url_for_site(app, site_id)?.trim_end_matches('/'), + path )) - })?; + .header(CONTROLLER_LOCAL_ONLY_HEADER, "1") + .json(body) + .send() + .await + .map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "failed to reach site controller for site `{site_id}` through site router: {err}" + )) + })?; if response.status().is_success() { return response.json().await.map_err(|err| { ProtocolApiError::control_state_unavailable(format!( - "site controller for site `{site_id}` returned invalid JSON: {err}" + "site controller for site `{site_id}` returned invalid JSON through site router: \ + {err}" )) }); } @@ -258,89 +345,36 @@ async fn forward_framework_request( return Err(protocol_error.into()); } Err(ProtocolApiError::control_state_unavailable(format!( - "site controller for site `{site_id}` returned {status}" - ))) -} - -async fn forward_framework_empty( - app: &SiteControllerApp, - site_id: &str, - method: Method, - path: &str, - record: &CapabilityInstanceRecord, - body: Option<&TReq>, -) -> std::result::Result<(), ProtocolApiError> { - let mut request = app - .control - .client - .request( - reqwest::Method::from_bytes(method.as_str().as_bytes()) - .expect("axum HTTP method should map to reqwest"), - format!( - "{}{}", - peer_controller_url(app, site_id)?.trim_end_matches('/'), - path - ), - ) - .header(FRAMEWORK_AUTH_HEADER, app.router_auth_token.as_ref()) - .header(FRAMEWORK_ROUTE_ID_HEADER, record.route_id.as_str()) - .header(FRAMEWORK_PEER_ID_HEADER, record.recipient_peer_id.as_str()); - if let Some(body) = body { - request = request.json(body); - } - let response = request.send().await.map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "failed to reach site controller for site `{site_id}`: {err}" - )) - })?; - if response.status().is_success() { - return Ok(()); - } - let status = response.status(); - let body = response.bytes().await.map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "failed to read site controller error response for site `{site_id}`: {err}" - )) - })?; - if let Ok(protocol_error) = serde_json::from_slice::(&body) { - return Err(protocol_error.into()); - } - Err(ProtocolApiError::control_state_unavailable(format!( - "site controller for site `{site_id}` returned {status}" + "site controller for site `{site_id}` returned {status} through site router" ))) } -async fn peer_dynamic_caps_post( +async fn peer_controller_get_via_router( app: &SiteControllerApp, site_id: &str, path: &str, - body: &TReq, ) -> std::result::Result { let response = app .control .client - .post(format!( + .get(format!( "{}{}", - peer_controller_url(app, site_id)?.trim_end_matches('/'), + peer_site_router_url_for_site(app, site_id)?.trim_end_matches('/'), path )) - .header( - FRAMEWORK_AUTH_HEADER, - app.control.control_state_auth_token.as_ref(), - ) .header(CONTROLLER_LOCAL_ONLY_HEADER, "1") - .json(body) .send() .await .map_err(|err| { ProtocolApiError::control_state_unavailable(format!( - "failed to reach site controller for site `{site_id}`: {err}" + "failed to reach site controller for site `{site_id}` through site router: {err}" )) })?; if response.status().is_success() { return response.json().await.map_err(|err| { ProtocolApiError::control_state_unavailable(format!( - "site controller for site `{site_id}` returned invalid JSON: {err}" + "site controller for site `{site_id}` returned invalid JSON through site router: \ + {err}" )) }); } @@ -354,10 +388,36 @@ async fn peer_dynamic_caps_post( return Err(protocol_error.into()); } Err(ProtocolApiError::control_state_unavailable(format!( - "site controller for site `{site_id}` returned {status}" + "site controller for site `{site_id}` returned {status} through site router" ))) } +pub(super) async fn peer_router_identity_via_router( + app: &SiteControllerApp, + site_id: &str, +) -> std::result::Result { + let response: RouterIdentityResponse = + peer_controller_get_via_router(app, site_id, "/v1/controller/router-identity").await?; + let public_key = base64::engine::general_purpose::STANDARD + .decode(response.public_key_b64) + .map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "site controller for site `{site_id}` returned an invalid router public key: {err}" + )) + })?; + let public_key: [u8; 32] = public_key.try_into().map_err(|_| { + ProtocolApiError::control_state_unavailable(format!( + "site controller for site `{site_id}` returned a router public key with the wrong \ + length" + )) + })?; + Ok(MeshIdentityPublic { + id: response.id, + public_key, + mesh_scope: Some(app.control.mesh_scope.to_string()), + }) +} + async fn local_held_list( app: &SiteControllerApp, request: ControlDynamicHeldListRequest, @@ -508,86 +568,11 @@ pub(super) async fn execute_site_controller_framework_inspect( ) -> std::result::Result { let authority_site = framework_request_site_id(state, record)?; if authority_site != app.control.controller_plan.site_id { - return match request { - FrameworkComponentInspectRequest::ListTemplates => { - Ok(FrameworkComponentInspectResponse::ListTemplates( - forward_framework_request( - app, - &authority_site, - Method::GET, - "/v1/templates", - record, - None::<&()>, - ) - .await?, - )) - } - FrameworkComponentInspectRequest::GetTemplate { template } => { - Ok(FrameworkComponentInspectResponse::GetTemplate( - forward_framework_request( - app, - &authority_site, - Method::GET, - &format!("/v1/templates/{template}"), - record, - None::<&()>, - ) - .await?, - )) - } - FrameworkComponentInspectRequest::ResolveTemplate { template, request } => { - Ok(FrameworkComponentInspectResponse::ResolveTemplate( - forward_framework_request( - app, - &authority_site, - Method::POST, - &format!("/v1/templates/{template}/resolve"), - record, - Some(&request), - ) - .await?, - )) - } - FrameworkComponentInspectRequest::ListChildren => { - Ok(FrameworkComponentInspectResponse::ListChildren( - forward_framework_request( - app, - &authority_site, - Method::GET, - "/v1/children", - record, - None::<&()>, - ) - .await?, - )) - } - FrameworkComponentInspectRequest::GetChild { child } => { - Ok(FrameworkComponentInspectResponse::GetChild( - forward_framework_request( - app, - &authority_site, - Method::GET, - &format!("/v1/children/{child}"), - record, - None::<&()>, - ) - .await?, - )) - } - FrameworkComponentInspectRequest::GetSnapshot => { - Ok(FrameworkComponentInspectResponse::GetSnapshot( - forward_framework_request( - app, - &authority_site, - Method::POST, - "/v1/snapshot", - record, - None::<&()>, - ) - .await?, - )) - } - }; + return Err(ProtocolApiError::control_state_unavailable(format!( + "framework.component request for authority site `{authority_site}` reached site \ + controller `{}`; router framework route overlays are stale", + app.control.controller_plan.site_id + ))); } ccs_api::execute_framework_component_inspect(state, record.authority_realm_id, request) @@ -603,38 +588,11 @@ pub(super) async fn execute_site_controller_framework_mutate( ) -> std::result::Result { let authority_site = framework_request_site_id(state, record)?; if authority_site != app.control.controller_plan.site_id { - return match request { - ccs_api::FrameworkComponentMutateRequest::CreateChild(request) => { - Ok(ccs_api::FrameworkComponentMutateResponse::CreateChild( - forward_framework_request( - app, - &authority_site, - Method::POST, - "/v1/children", - record, - Some(&request), - ) - .await?, - )) - } - ccs_api::FrameworkComponentMutateRequest::DestroyChild { child } => { - forward_framework_empty::<()>( - app, - &authority_site, - Method::DELETE, - &format!("/v1/children/{child}"), - record, - None, - ) - .await?; - Ok(ccs_api::FrameworkComponentMutateResponse::DestroyChild( - ccs_api::DestroyChildResponse { - child, - destroyed: true, - }, - )) - } - }; + return Err(ProtocolApiError::control_state_unavailable(format!( + "framework.component request for authority site `{authority_site}` reached site \ + controller `{}`; router framework route overlays are stale", + app.control.controller_plan.site_id + ))); } match request { @@ -668,11 +626,11 @@ pub(super) async fn execute_site_controller_dynamic_caps_inspect( )); } let mut held = local_held_list(app, request.clone()).await?.held; - for site_id in app.control.peer_controllers.keys() { + for site_id in peer_site_ids(app) { held.extend( - peer_dynamic_caps_post::<_, HeldListResponse>( + peer_dynamic_caps_post_via_router::<_, HeldListResponse>( app, - site_id, + &site_id, "/v1/controller/dynamic-caps/held", &request, ) @@ -699,7 +657,7 @@ pub(super) async fn execute_site_controller_dynamic_caps_inspect( )); } Ok(DynamicCapsInspectResponse::HeldDetail( - peer_dynamic_caps_post( + peer_dynamic_caps_post_via_router( app, &site_id, "/v1/controller/dynamic-caps/held/detail", @@ -722,7 +680,7 @@ pub(super) async fn execute_site_controller_dynamic_caps_inspect( )); } Ok(DynamicCapsInspectResponse::InspectRef( - peer_dynamic_caps_post( + peer_dynamic_caps_post_via_router( app, &site_id, "/v1/controller/dynamic-caps/inspect-ref", @@ -744,12 +702,22 @@ pub(super) async fn execute_site_controller_dynamic_caps_inspect( local_resolve_origin(app, request).await?, )); } + let holder_runtime = + local_component_runtime(&app.control, &state, &request.holder_component_id) + .map_err(ProtocolApiError::from)?; Ok(DynamicCapsInspectResponse::ResolveOrigin( - peer_dynamic_caps_post( + peer_dynamic_caps_post_via_router( app, &site_id, - "/v1/controller/dynamic-caps/resolve-origin", - &request, + "/v1/internal/dynamic-caps/resolve-origin", + &InternalDynamicResolveOriginRequest { + holder_component_id: request.holder_component_id, + source: request.source, + holder_peer_id: holder_runtime.mesh_config.identity.id.clone(), + holder_peer_key_b64: base64::engine::general_purpose::STANDARD + .encode(holder_runtime.mesh_config.identity.public_key), + holder_site_kind: app.control.controller_plan.kind, + }, ) .await?, )) @@ -777,7 +745,7 @@ pub(super) async fn execute_site_controller_dynamic_caps_mutate( )); } Ok(DynamicCapsMutateResponse::Share( - peer_dynamic_caps_post( + peer_dynamic_caps_post_via_router( app, &site_id, "/v1/controller/dynamic-caps/share", @@ -800,7 +768,7 @@ pub(super) async fn execute_site_controller_dynamic_caps_mutate( )); } Ok(DynamicCapsMutateResponse::Revoke( - peer_dynamic_caps_post( + peer_dynamic_caps_post_via_router( app, &site_id, "/v1/controller/dynamic-caps/revoke", @@ -965,7 +933,7 @@ async fn publish_dynamic_origin( Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + authorize_local_controller_request(&app, &headers)?; let ccs_app = LocalDynamicCapabilityOriginApp { site_state_root: PathBuf::from(&app.control.controller_plan.site_state_root), runtime: app.control.runtime.clone(), @@ -975,37 +943,88 @@ async fn publish_dynamic_origin( )) } -async fn control_create_child_route( +async fn resolve_dynamic_origin_internal_route( State(app): State, headers: HeaderMap, - Json(request): Json, -) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; - let authority_realm_id = { - let state = app.control.control_state.lock().await; - capability_instance_record(&state, &request.cap_instance_id) - .map_err(ProtocolApiError::from)? - .authority_realm_id - }; + Json(request): Json, +) -> std::result::Result, ProtocolApiError> +{ + authorize_local_controller_request(&app, &headers)?; Ok(Json( - execute_create_child(&app.control, authority_realm_id, request.request).await?, + resolve_dynamic_capability_origin_internal(&app.control, request).await?, )) } -async fn control_destroy_child_route( +async fn revoke_dynamic_origin_overlays( State(app): State, headers: HeaderMap, - AxumPath(child): AxumPath, - Json(request): Json, + Json(request): Json, ) -> std::result::Result { - authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; - let authority_realm_id = { - let state = app.control.control_state.lock().await; - capability_instance_record(&state, &request.cap_instance_id) - .map_err(ProtocolApiError::from)? - .authority_realm_id - }; - execute_destroy_child(&app.control, authority_realm_id, &child).await?; + authorize_local_controller_request(&app, &headers)?; + revoke_dynamic_capability_origin_overlays_local(&app.control, &request) + .await + .map_err(ProtocolApiError::from)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn resolve_external_link_url( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + authorize_local_controller_request(&app, &headers)?; + Ok(Json( + resolve_external_link_url_local(&app.control, &request) + .await + .map_err(ProtocolApiError::from)?, + )) +} + +async fn publish_external_slot_overlay_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize_local_controller_request(&app, &headers)?; + publish_external_slot_overlay_local(&app.control, &request) + .await + .map_err(ProtocolApiError::from)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn clear_external_slot_overlay_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize_local_controller_request(&app, &headers)?; + clear_external_slot_overlay_local(&app.control, &request) + .await + .map_err(ProtocolApiError::from)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn publish_export_peer_overlay_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize_local_controller_request(&app, &headers)?; + publish_export_peer_overlay_local(&app.control, &request) + .await + .map_err(ProtocolApiError::from)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn clear_export_peer_overlay_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize_local_controller_request(&app, &headers)?; + clear_export_peer_overlay_local(&app.control, &request) + .await + .map_err(ProtocolApiError::from)?; Ok(StatusCode::NO_CONTENT) } @@ -1117,86 +1136,3 @@ async fn control_dynamic_resolve_origin_route( _ => unreachable!("resolve_origin should return origin resolution"), } } - -async fn site_prepare_child_route( - State(app): State, - headers: HeaderMap, - AxumPath(child_id): AxumPath, - Json(request): Json, -) -> std::result::Result { - authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; - app.control - .runtime - .prepare_child(&app.control.controller_plan, child_id, request.site_plan) - .await - .map_err(|err| { - ProtocolApiError::from(protocol_error( - ProtocolErrorCode::PrepareFailed, - &err.to_string(), - )) - })?; - Ok(StatusCode::NO_CONTENT) -} - -async fn site_publish_child_route( - State(app): State, - headers: HeaderMap, - AxumPath(child_id): AxumPath, - Json(request): Json, -) -> std::result::Result { - authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; - app.control - .runtime - .publish_child(&app.control.controller_plan, child_id, request.site_plan) - .await - .map_err(|err| { - ProtocolApiError::from(protocol_error( - ProtocolErrorCode::PublishFailed, - &err.to_string(), - )) - })?; - Ok(StatusCode::NO_CONTENT) -} - -async fn site_rollback_child_route( - State(app): State, - headers: HeaderMap, - AxumPath(child_id): AxumPath, -) -> std::result::Result { - authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; - app.control - .runtime - .rollback_child(&app.control.controller_plan, child_id) - .await - .map_err(|err| { - ProtocolApiError::from(protocol_error( - ProtocolErrorCode::PublishFailed, - &err.to_string(), - )) - })?; - Ok(StatusCode::NO_CONTENT) -} - -async fn site_destroy_child_route( - State(app): State, - headers: HeaderMap, - AxumPath(child_id): AxumPath, - Json(request): Json, -) -> std::result::Result { - authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; - app.control - .runtime - .destroy_child( - &app.control.controller_plan, - child_id, - request.desired_site_plan, - ) - .await - .map_err(|err| { - ProtocolApiError::from(protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &err.to_string(), - )) - })?; - Ok(StatusCode::NO_CONTENT) -} diff --git a/runtime/site-controller/src/state.rs b/runtime/site-controller/src/state.rs index 1b07ce71..ff2b760d 100644 --- a/runtime/site-controller/src/state.rs +++ b/runtime/site-controller/src/state.rs @@ -9,6 +9,13 @@ pub(super) const SITE_CONTROLLER_STATE_PATH: &str = "/v1/controller/state"; pub(super) const FRAMEWORK_ROUTE_ID_HEADER: &str = "x-amber-route-id"; pub(super) const FRAMEWORK_PEER_ID_HEADER: &str = "x-amber-peer-id"; pub(super) const FRAMEWORK_AUTH_HEADER: &str = "x-amber-framework-auth"; +pub const SITE_CONTROLLER_INTERNAL_CAPABILITY: &str = "amber.internal.site_controller"; +pub const SITE_CONTROLLER_SERVICE_NAME: &str = "amber-site-controller"; +pub const SITE_CONTROLLER_PORT: u16 = 4100; + +pub fn site_controller_internal_route_id(site_id: &str) -> String { + format!("site-controller:{site_id}") +} #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct FrozenPlacementState { @@ -62,8 +69,6 @@ pub(crate) struct LiveChildRecord { #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub(crate) assignments: BTreeMap, #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub(crate) site_plans: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] pub(crate) overlay_ids: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub(crate) overlays: Vec, @@ -84,22 +89,16 @@ pub(crate) struct PendingDestroyRecord { } #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct DynamicSitePlanRecord { - pub site_id: String, - pub kind: SiteKind, - pub router_identity_id: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub component_ids: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub assigned_components: Vec, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub artifact_files: BTreeMap, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub desired_artifact_files: BTreeMap, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub proxy_exports: BTreeMap, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub routed_inputs: Vec, +pub struct DynamicInputDirectRecord { + pub component: String, + pub slot: String, + pub provider_component: String, + pub protocol: String, + pub capability_kind: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub capability_profile: Option, + #[serde(flatten)] + pub target: DynamicInputRouteTarget, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -251,13 +250,6 @@ fn default_framework_id_stride() -> u64 { 1 } -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct SiteControllerPeerPlan { - pub site_id: String, - pub kind: SiteKind, - pub authority_url: String, -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub struct SiteControllerPlan { pub schema: String, @@ -269,6 +261,14 @@ pub struct SiteControllerPlan { pub listen_addr: SocketAddr, pub authority_url: String, pub router_identity_id: String, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub peer_site_router_urls: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub peer_router_mesh_addrs: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub local_router_control: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub published_router_mesh_addr: Option, pub state_path: String, pub run_root: String, pub state_root: String, @@ -276,8 +276,6 @@ pub struct SiteControllerPlan { pub artifact_dir: String, pub auth_token: String, pub dynamic_caps_token_verify_key_b64: String, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub peer_controllers: BTreeMap, #[serde(default, skip_serializing_if = "Option::is_none")] pub storage_root: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -795,7 +793,6 @@ pub(super) fn restore_framework_children_from_snapshot( fragment: Some(child.fragment), input_bindings: child.input_bindings, assignments: child.assignments, - site_plans: Vec::new(), overlay_ids: Vec::new(), overlays: Vec::new(), outputs: child.outputs, @@ -1206,6 +1203,10 @@ pub fn write_site_controller_plan( listen_addr: SocketAddr, authority_url: &str, router_identity_id: &str, + peer_site_router_urls: &BTreeMap, + peer_router_mesh_addrs: &BTreeMap, + local_router_control: Option<&str>, + published_router_mesh_addr: Option<&str>, state_path: &Path, run_root: &Path, state_root: &Path, @@ -1213,7 +1214,6 @@ pub fn write_site_controller_plan( artifact_dir: &Path, auth_token: &str, dynamic_caps_token_verify_key_b64: &str, - peer_controllers: BTreeMap, storage_root: Option<&str>, runtime_root: Option<&str>, router_mesh_port: Option, @@ -1233,6 +1233,10 @@ pub fn write_site_controller_plan( listen_addr, authority_url: authority_url.to_string(), router_identity_id: router_identity_id.to_string(), + peer_site_router_urls: peer_site_router_urls.clone(), + peer_router_mesh_addrs: peer_router_mesh_addrs.clone(), + local_router_control: local_router_control.map(str::to_string), + published_router_mesh_addr: published_router_mesh_addr.map(str::to_string), state_path: state_path.display().to_string(), run_root: run_root.display().to_string(), state_root: state_root.display().to_string(), @@ -1240,7 +1244,6 @@ pub fn write_site_controller_plan( artifact_dir: artifact_dir.display().to_string(), auth_token: auth_token.to_string(), dynamic_caps_token_verify_key_b64: dynamic_caps_token_verify_key_b64.to_string(), - peer_controllers, storage_root: storage_root.map(str::to_string), runtime_root: runtime_root.map(str::to_string), router_mesh_port, diff --git a/runtime/site-controller/src/tests.rs b/runtime/site-controller/src/tests.rs index bf6c4823..63bc79f2 100644 --- a/runtime/site-controller/src/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -1,4 +1,10 @@ -use std::fs; +use std::{ + fs, + future::Future, + io::{Read, Write}, + sync::OnceLock, + time::Duration as StdDuration, +}; use amber_compiler::run_plan::build_run_plan; use amber_mesh::{ @@ -10,7 +16,7 @@ use amber_mesh::{ HeldEntryKind, HeldEntryState, RootAuthoritySelectorIr, }, }; -use axum::{Router, extract::State, http::HeaderMap, routing::post}; +use axum::{Router, http::HeaderMap}; use reqwest::{Client, StatusCode}; use serde::{Serialize, de::DeserializeOwned}; use serde_json::{Value, json}; @@ -18,6 +24,11 @@ use tempfile::TempDir; use url::Url; use super::{api::*, http::*, orchestration::*, planner::*, state::*, *}; +use crate::{ + ccs_api::FrameworkComponentInspectRequest, + runtime_api::{SharedSiteControllerRuntime, SiteControllerRuntime}, + site_controller::RouterIdentityResponse, +}; #[derive(Clone, Debug, Default, Serialize, Deserialize)] struct DirectRuntimeState { @@ -64,8 +75,8 @@ impl SiteControllerRuntime for TestSiteControllerRuntime { fn prepare_child<'a>( &'a self, _plan: &'a SiteControllerPlan, - _child_id: u64, - _site_plan: DynamicSitePlanRecord, + _state: FrameworkControlState, + _child: LiveChildRecord, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -73,8 +84,8 @@ impl SiteControllerRuntime for TestSiteControllerRuntime { fn publish_child<'a>( &'a self, _plan: &'a SiteControllerPlan, - _child_id: u64, - _site_plan: DynamicSitePlanRecord, + _state: FrameworkControlState, + _child: LiveChildRecord, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -90,8 +101,8 @@ impl SiteControllerRuntime for TestSiteControllerRuntime { fn destroy_child<'a>( &'a self, _plan: &'a SiteControllerPlan, - _child_id: u64, - _desired_site_plan: Option, + _state: FrameworkControlState, + _child: LiveChildRecord, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -148,6 +159,7 @@ impl SiteControllerRuntime for TestSiteControllerRuntime { LiveComponentRuntimeMetadata { moniker: moniker.to_string(), host_mesh_addr: format!("127.0.0.1:{port}"), + control_endpoint: None, mesh_config, }, ); @@ -228,7 +240,8 @@ fn test_runtime() -> SharedSiteControllerRuntime { Arc::new(TestSiteControllerRuntime) } -type DestroyCalls = Arc)>>>; +type DestroyCalls = Arc>>; +type PublishCalls = Arc>>; #[derive(Clone, Default)] struct FailingPublishRuntime { @@ -254,8 +267,8 @@ impl SiteControllerRuntime for FailingPublishRuntime { fn prepare_child<'a>( &'a self, _plan: &'a SiteControllerPlan, - _child_id: u64, - _site_plan: DynamicSitePlanRecord, + _state: FrameworkControlState, + _child: LiveChildRecord, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -263,8 +276,8 @@ impl SiteControllerRuntime for FailingPublishRuntime { fn publish_child<'a>( &'a self, _plan: &'a SiteControllerPlan, - _child_id: u64, - _site_plan: DynamicSitePlanRecord, + _state: FrameworkControlState, + _child: LiveChildRecord, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Err(miette::miette!("publish exploded")) }) } @@ -280,15 +293,16 @@ impl SiteControllerRuntime for FailingPublishRuntime { fn destroy_child<'a>( &'a self, _plan: &'a SiteControllerPlan, - child_id: u64, - desired_site_plan: Option, + _state: FrameworkControlState, + child: LiveChildRecord, ) -> SiteControllerRuntimeFuture<'a, ()> { let destroy_calls = self.destroy_calls.clone(); Box::pin(async move { + let site_id = child_runtime_site_id(&child).expect("child site id"); destroy_calls .lock() .expect("destroy call log mutex should lock") - .push((child_id, desired_site_plan)); + .push((child.child_id, site_id)); Ok(()) }) } @@ -353,6 +367,276 @@ impl SiteControllerRuntime for FailingPublishRuntime { } } +#[derive(Clone, Default)] +struct RecordingPublishRuntime { + publish_calls: PublishCalls, +} + +impl SiteControllerRuntime for RecordingPublishRuntime { + fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()> { + TestSiteControllerRuntime.cleanup() + } + + fn resolve_link_external_url<'a>( + &'a self, + provider: &'a LaunchedSite, + provider_output_dir: &'a Path, + link: &'a amber_compiler::run_plan::RunLink, + consumer_kind: SiteKind, + run_root: &'a Path, + ) -> SiteControllerRuntimeFuture<'a, String> { + TestSiteControllerRuntime.resolve_link_external_url( + provider, + provider_output_dir, + link, + consumer_kind, + run_root, + ) + } + + fn prepare_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn publish_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + child: LiveChildRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + let publish_calls = self.publish_calls.clone(); + Box::pin(async move { + let site_id = child_runtime_site_id(&child).expect("child site id"); + publish_calls + .lock() + .expect("publish call log mutex should lock") + .push((child.child_id, site_id)); + Ok(()) + }) + } + + fn rollback_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _child_id: u64, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn destroy_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn collect_live_component_runtime_metadata( + &self, + plan: &SiteControllerRuntimePlan, + ) -> miette::Result> { + TestSiteControllerRuntime.collect_live_component_runtime_metadata(plan) + } + + fn load_live_site_router_mesh_config( + &self, + plan: &SiteControllerRuntimePlan, + ) -> miette::Result { + TestSiteControllerRuntime.load_live_site_router_mesh_config(plan) + } + + fn router_mesh_addr_for_consumer( + &self, + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> miette::Result { + TestSiteControllerRuntime.router_mesh_addr_for_consumer( + provider_kind, + consumer_kind, + router_mesh_addr, + ) + } + + fn update_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExternalSlotOverlay, + ) -> miette::Result<()> { + TestSiteControllerRuntime.update_desired_overlay_for_consumer( + site_state_root, + overlay_id, + overlay, + ) + } + + fn update_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExportPeerOverlay, + ) -> miette::Result<()> { + TestSiteControllerRuntime.update_desired_overlay_for_provider( + site_state_root, + overlay_id, + overlay, + ) + } + + fn clear_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> miette::Result<()> { + TestSiteControllerRuntime.clear_desired_overlay_for_consumer(site_state_root, overlay_id) + } + + fn clear_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> miette::Result<()> { + TestSiteControllerRuntime.clear_desired_overlay_for_provider(site_state_root, overlay_id) + } +} + +#[derive(Clone, Default)] +struct FailingRollbackRuntime; + +impl SiteControllerRuntime for FailingRollbackRuntime { + fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()> { + TestSiteControllerRuntime.cleanup() + } + + fn resolve_link_external_url<'a>( + &'a self, + provider: &'a LaunchedSite, + provider_output_dir: &'a Path, + link: &'a amber_compiler::run_plan::RunLink, + consumer_kind: SiteKind, + run_root: &'a Path, + ) -> SiteControllerRuntimeFuture<'a, String> { + TestSiteControllerRuntime.resolve_link_external_url( + provider, + provider_output_dir, + link, + consumer_kind, + run_root, + ) + } + + fn prepare_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn publish_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn rollback_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _child_id: u64, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Err(miette::miette!("rollback exploded")) }) + } + + fn destroy_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn collect_live_component_runtime_metadata( + &self, + plan: &SiteControllerRuntimePlan, + ) -> miette::Result> { + TestSiteControllerRuntime.collect_live_component_runtime_metadata(plan) + } + + fn load_live_site_router_mesh_config( + &self, + plan: &SiteControllerRuntimePlan, + ) -> miette::Result { + TestSiteControllerRuntime.load_live_site_router_mesh_config(plan) + } + + fn router_mesh_addr_for_consumer( + &self, + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> miette::Result { + TestSiteControllerRuntime.router_mesh_addr_for_consumer( + provider_kind, + consumer_kind, + router_mesh_addr, + ) + } + + fn update_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExternalSlotOverlay, + ) -> miette::Result<()> { + TestSiteControllerRuntime.update_desired_overlay_for_consumer( + site_state_root, + overlay_id, + overlay, + ) + } + + fn update_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExportPeerOverlay, + ) -> miette::Result<()> { + TestSiteControllerRuntime.update_desired_overlay_for_provider( + site_state_root, + overlay_id, + overlay, + ) + } + + fn clear_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> miette::Result<()> { + TestSiteControllerRuntime.clear_desired_overlay_for_consumer(site_state_root, overlay_id) + } + + fn clear_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> miette::Result<()> { + TestSiteControllerRuntime.clear_desired_overlay_for_provider(site_state_root, overlay_id) + } +} + fn with_runtime(app: &ControlStateApp, runtime: SharedSiteControllerRuntime) -> ControlStateApp { ControlStateApp { control_state: app.control_state.clone(), @@ -363,7 +647,6 @@ fn with_runtime(app: &ControlStateApp, runtime: SharedSiteControllerRuntime) -> mesh_scope: app.mesh_scope.clone(), control_state_auth_token: app.control_state_auth_token.clone(), controller_plan: app.controller_plan.clone(), - peer_controllers: app.peer_controllers.clone(), authority_locks: app.authority_locks.clone(), runtime, } @@ -800,6 +1083,7 @@ fn test_live_component_runtime( LiveComponentRuntimeMetadata { moniker: moniker.to_string(), host_mesh_addr: host_mesh_addr.to_string(), + control_endpoint: None, mesh_config: MeshConfigPublic { identity: MeshIdentityPublic { id: peer_id.to_string(), @@ -1119,6 +1403,7 @@ fn dynamic_capability_origin_target_mesh_peer_uses_runtime_peer_catalog_for_bind let runtime = LiveComponentRuntimeMetadata { moniker: "/consumer".to_string(), host_mesh_addr: "127.0.0.1:24002".to_string(), + control_endpoint: None, mesh_config: MeshConfigPublic { identity: MeshIdentityPublic { id: "/consumer".to_string(), @@ -1381,7 +1666,6 @@ async fn same_site_dynamic_child_output_bindings_reuse_provider_component_routes write_control_state(&state_path, &state).expect("state should write"); let root_authority = state.base_scenario.root; let app = test_control_state_app(&dir, state, state_path); - let (app, controllers) = install_success_site_controllers(&app).await; execute_create_child( &app, @@ -1422,26 +1706,27 @@ async fn same_site_dynamic_child_output_bindings_reuse_provider_component_routes .iter() .find(|child| child.name == "consumer") .expect("consumer child should be recorded"); - for site_plan in &consumer.site_plans { - assert_eq!(site_plan.routed_inputs.len(), 1); - assert_eq!(site_plan.routed_inputs[0].component, "/consumer"); - assert_eq!(site_plan.routed_inputs[0].slot, "required_api"); - assert_eq!(site_plan.routed_inputs[0].provider_component, "/required"); - assert_eq!(site_plan.routed_inputs[0].protocol, "http"); - assert_eq!(site_plan.routed_inputs[0].capability_kind, "http"); - assert_eq!( - site_plan.routed_inputs[0].target, - DynamicInputRouteTarget::ComponentProvide { - provide: "http".to_string() - }, - "same-site child exports should reuse the provider component route instead of \ - inventing a synthetic dynamic-export hop", - ); - } - - for controller in controllers { - controller.abort(); - } + let site_id = child_runtime_site_id(consumer).expect("consumer site id"); + let runtime_spec = + build_local_child_runtime_spec(&state, consumer, &site_id).expect("runtime spec"); + assert_eq!(runtime_spec.direct_inputs.len(), 1); + assert!(runtime_spec.routed_inputs.is_empty()); + assert_eq!(runtime_spec.direct_inputs[0].component, "/consumer"); + assert_eq!(runtime_spec.direct_inputs[0].slot, "required_api"); + assert_eq!( + runtime_spec.direct_inputs[0].provider_component, + "/required" + ); + assert_eq!(runtime_spec.direct_inputs[0].protocol, "http"); + assert_eq!(runtime_spec.direct_inputs[0].capability_kind, "http"); + assert_eq!( + runtime_spec.direct_inputs[0].target, + DynamicInputRouteTarget::ComponentProvide { + provide: "http".to_string() + }, + "same-site child exports should reuse the provider component route without routing \ + through the site router", + ); } #[tokio::test] @@ -1539,7 +1824,6 @@ async fn same_site_static_child_export_bindings_reuse_provider_component_routes( write_control_state(&state_path, &state).expect("state should write"); let root_authority = state.base_scenario.root; let app = test_control_state_app(&dir, state, state_path); - let (app, controllers) = install_success_site_controllers(&app).await; execute_create_child( &app, @@ -1567,25 +1851,27 @@ async fn same_site_static_child_export_bindings_reuse_provider_component_routes( .iter() .find(|child| child.name == "consumer") .expect("consumer child should be recorded"); - for site_plan in &consumer.site_plans { - assert_eq!(site_plan.routed_inputs.len(), 1); - assert_eq!(site_plan.routed_inputs[0].component, "/consumer"); - assert_eq!(site_plan.routed_inputs[0].slot, "upstream"); - assert_eq!(site_plan.routed_inputs[0].provider_component, "/provider"); - assert_eq!(site_plan.routed_inputs[0].protocol, "http"); - assert_eq!(site_plan.routed_inputs[0].capability_kind, "http"); - assert_eq!( - site_plan.routed_inputs[0].target, - DynamicInputRouteTarget::ComponentProvide { - provide: "http".to_string() - }, - "same-site static child exports should reuse the provider component route", - ); - } - - for controller in controllers { - controller.abort(); - } + let site_id = child_runtime_site_id(consumer).expect("consumer site id"); + let runtime_spec = + build_local_child_runtime_spec(&state, consumer, &site_id).expect("runtime spec"); + assert_eq!(runtime_spec.direct_inputs.len(), 1); + assert!(runtime_spec.routed_inputs.is_empty()); + assert_eq!(runtime_spec.direct_inputs[0].component, "/consumer"); + assert_eq!(runtime_spec.direct_inputs[0].slot, "upstream"); + assert_eq!( + runtime_spec.direct_inputs[0].provider_component, + "/provider" + ); + assert_eq!(runtime_spec.direct_inputs[0].protocol, "http"); + assert_eq!(runtime_spec.direct_inputs[0].capability_kind, "http"); + assert_eq!( + runtime_spec.direct_inputs[0].target, + DynamicInputRouteTarget::ComponentProvide { + provide: "http".to_string() + }, + "same-site static child exports should reuse the provider component route without routing \ + through the site router", + ); } #[test] @@ -1836,6 +2122,7 @@ fn empty_live_child( child_id: u64, state: ChildState, ) -> LiveChildRecord { + let child_moniker = format!("/{name}"); LiveChildRecord { child_id, authority_realm_id, @@ -1843,10 +2130,29 @@ fn empty_live_child( state, template_name: Some("worker".to_string()), selected_manifest_catalog_key: None, - fragment: None, + fragment: Some(LiveScenarioFragment { + root_component_id: child_id as usize + 10_000, + components: vec![ComponentIr { + id: child_id as usize + 10_000, + moniker: child_moniker.clone(), + parent: Some(authority_realm_id), + children: Vec::new(), + resolved_url: Some(format!("file:///tmp/{name}.json5")), + digest: amber_manifest::ManifestDigest::new([0; 32]), + config: None, + config_schema: None, + program: None, + slots: BTreeMap::new(), + provides: BTreeMap::new(), + exports: BTreeMap::new(), + resources: BTreeMap::new(), + child_templates: BTreeMap::new(), + metadata: None, + }], + bindings: Vec::new(), + }), input_bindings: Vec::new(), - assignments: BTreeMap::new(), - site_plans: Vec::new(), + assignments: BTreeMap::from([(child_moniker, "direct_local".to_string())]), overlay_ids: Vec::new(), overlays: Vec::new(), outputs: BTreeMap::new(), @@ -1861,6 +2167,67 @@ fn pending_destroy(tx_id: u64, child: LiveChildRecord) -> PendingDestroyRecord { PendingDestroyRecord { tx_id, child } } +fn test_router_control_addr() -> String { + static ROUTER_CONTROL_ADDR: OnceLock = OnceLock::new(); + ROUTER_CONTROL_ADDR + .get_or_init(|| { + let listener = std::net::TcpListener::bind(("127.0.0.1", 0)) + .expect("mock router control listener should bind"); + let addr = listener + .local_addr() + .expect("mock router control listener addr"); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { + continue; + }; + stream + .set_read_timeout(Some(StdDuration::from_millis(100))) + .expect("mock router control read timeout should set"); + let mut request = Vec::new(); + let mut buf = [0u8; 4096]; + loop { + match stream.read(&mut buf) { + Ok(0) => break, + Ok(read) => request.extend_from_slice(&buf[..read]), + Err(err) + if matches!( + err.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) => + { + break; + } + Err(_) => break, + } + } + let request = String::from_utf8_lossy(&request); + let response = if request.starts_with("GET /identity ") { + let body = serde_json::to_string(&json!({ + "id": "/site/test/router", + "public_key": vec![9u8; 32], + "mesh_scope": "test-mesh", + })) + .expect("mock router identity should serialize"); + format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: \ + {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ) + } else { + "HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + .to_string() + }; + let _ = stream.write_all(response.as_bytes()); + let _ = stream.flush(); + } + }); + addr.to_string() + }) + .clone() +} + fn test_control_state_app( dir: &TempDir, state: FrameworkControlState, @@ -1870,13 +2237,49 @@ fn test_control_state_app( let state_root = dir.path().join("state"); fs::create_dir_all(&run_root).expect("run root should exist"); fs::create_dir_all(&state_root).expect("state root should exist"); - let site_id = state - .placement - .offered_sites + let offered_sites = if state.placement.offered_sites.is_empty() { + BTreeMap::from([( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + )]) + } else { + state.placement.offered_sites.clone() + }; + let router_control = test_router_control_addr(); + let router_public_key_b64 = base64::engine::general_purpose::STANDARD.encode([7u8; 32]); + let site_id = offered_sites .keys() .next() .cloned() - .unwrap_or_else(|| "direct_local".to_string()); + .expect("offered sites should contain at least one site"); + for (site_id, site_definition) in &offered_sites { + let site_state_root = state_root.join(site_id); + let artifact_dir = dir.path().join("artifact").join(site_id); + let storage_root = dir.path().join("storage").join(site_id); + let runtime_root = dir.path().join("runtime").join(site_id); + fs::create_dir_all(&site_state_root).expect("site state root should exist"); + fs::create_dir_all(&artifact_dir).expect("artifact dir should exist"); + fs::create_dir_all(&storage_root).expect("storage root should exist"); + fs::create_dir_all(&runtime_root).expect("runtime root should exist"); + write_json( + &site_state_root.join("manager-state.json"), + &json!({ + "status": "running", + "kind": site_definition.kind, + "artifact_dir": artifact_dir.display().to_string(), + "supervisor_pid": 1u32, + "router_control": router_control.clone(), + "router_mesh_addr": "127.0.0.1:24000", + "router_identity_id": format!("/site/{site_id}/router"), + "router_public_key_b64": router_public_key_b64.clone(), + "site_controller_url": "http://127.0.0.1:0", + }), + ) + .expect("site manager state should write"); + } let site_state_root = state_root.join(&site_id); let artifact_dir = dir.path().join("artifact").join(&site_id); let storage_root = dir.path().join("storage").join(&site_id); @@ -1899,6 +2302,10 @@ fn test_control_state_app( listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), authority_url: "http://127.0.0.1:0".to_string(), router_identity_id: format!("/site/{site_id}/router"), + peer_site_router_urls: BTreeMap::new(), + peer_router_mesh_addrs: BTreeMap::new(), + local_router_control: None, + published_router_mesh_addr: Some("127.0.0.1:24000".to_string()), state_path: state_path.display().to_string(), run_root: run_root.display().to_string(), state_root: state_root.display().to_string(), @@ -1906,7 +2313,6 @@ fn test_control_state_app( artifact_dir: artifact_dir.display().to_string(), auth_token: "test-control-state-auth".to_string(), dynamic_caps_token_verify_key_b64: String::new(), - peer_controllers: BTreeMap::new(), storage_root: Some(storage_root.display().to_string()), runtime_root: Some(runtime_root.display().to_string()), router_mesh_port: Some(24000), @@ -1916,7 +2322,6 @@ fn test_control_state_app( observability_endpoint: None, launch_env: BTreeMap::new(), }), - peer_controllers: Arc::new(BTreeMap::new()), authority_locks: Arc::new(Mutex::new(BTreeMap::new())), runtime: test_runtime(), } @@ -1976,13 +2381,14 @@ async fn http_get_json( url: &str, headers: &[(String, String)], ) -> T { - let response = apply_headers(client.get(url), headers) - .send() - .await - .unwrap_or_else(|err| panic!("send GET {url}: {err}")); + let response = with_test_timeout( + format!("GET {url}"), + apply_headers(client.get(url), headers).send(), + ) + .await + .unwrap_or_else(|err| panic!("send GET {url}: {err}")); let status = response.status(); - let body = response - .text() + let body = with_test_timeout(format!("read GET {url}"), response.text()) .await .unwrap_or_else(|err| panic!("read GET {url}: {err}")); assert_eq!(status, StatusCode::OK, "GET {url} failed: {body}"); @@ -1996,14 +2402,14 @@ async fn http_post_json( headers: &[(String, String)], body: &Req, ) -> T { - let response = apply_headers(client.post(url), headers) - .json(body) - .send() - .await - .unwrap_or_else(|err| panic!("send POST {url}: {err}")); + let response = with_test_timeout( + format!("POST {url}"), + apply_headers(client.post(url), headers).json(body).send(), + ) + .await + .unwrap_or_else(|err| panic!("send POST {url}: {err}")); let status = response.status(); - let body = response - .text() + let body = with_test_timeout(format!("read POST {url}"), response.text()) .await .unwrap_or_else(|err| panic!("read POST {url}: {err}")); assert_eq!(status, StatusCode::OK, "POST {url} failed: {body}"); @@ -2016,13 +2422,14 @@ async fn http_post_empty_json( url: &str, headers: &[(String, String)], ) -> T { - let response = apply_headers(client.post(url), headers) - .send() - .await - .unwrap_or_else(|err| panic!("send POST {url}: {err}")); + let response = with_test_timeout( + format!("POST {url}"), + apply_headers(client.post(url), headers).send(), + ) + .await + .unwrap_or_else(|err| panic!("send POST {url}: {err}")); let status = response.status(); - let body = response - .text() + let body = with_test_timeout(format!("read POST {url}"), response.text()) .await .unwrap_or_else(|err| panic!("read POST {url}: {err}")); assert_eq!(status, StatusCode::OK, "POST {url} failed: {body}"); @@ -2031,13 +2438,14 @@ async fn http_post_empty_json( } async fn http_delete_empty(client: &Client, url: &str, headers: &[(String, String)]) { - let response = apply_headers(client.delete(url), headers) - .send() - .await - .unwrap_or_else(|err| panic!("send DELETE {url}: {err}")); + let response = with_test_timeout( + format!("DELETE {url}"), + apply_headers(client.delete(url), headers).send(), + ) + .await + .unwrap_or_else(|err| panic!("send DELETE {url}: {err}")); let status = response.status(); - let body = response - .text() + let body = with_test_timeout(format!("read DELETE {url}"), response.text()) .await .unwrap_or_else(|err| panic!("read DELETE {url}: {err}")); assert_eq!( @@ -2071,6 +2479,14 @@ fn normalize_dynamic_share_ref(value: &mut Value) { const TEST_SITE_STATE_SCHEMA: &str = "amber.run.site_state"; const TEST_SITE_STATE_VERSION: u32 = 2; +const TEST_REQUEST_TIMEOUT: StdDuration = StdDuration::from_secs(15); + +async fn with_test_timeout(label: impl Into, future: impl Future) -> T { + let label = label.into(); + tokio::time::timeout(TEST_REQUEST_TIMEOUT, future) + .await + .unwrap_or_else(|_| panic!("{label} timed out after {:?}", TEST_REQUEST_TIMEOUT)) +} fn with_controller_endpoint( app: &ControlStateApp, @@ -2089,7 +2505,6 @@ fn with_controller_endpoint( mesh_scope: app.mesh_scope.clone(), control_state_auth_token: app.control_state_auth_token.clone(), controller_plan: Arc::new(controller_plan), - peer_controllers: app.peer_controllers.clone(), authority_locks: app.authority_locks.clone(), runtime: app.runtime.clone(), } @@ -2216,16 +2631,24 @@ impl TestMcpClient { }, }, }); - let response = apply_headers(client.post(endpoint), &headers) - .header("content-type", "application/json") - .header("accept", "application/json, text/event-stream") - .json(&initialize) - .send() - .await - .expect("send initialize request"); + let response = with_test_timeout( + format!("MCP initialize request to {endpoint}"), + apply_headers(client.post(endpoint), &headers) + .header("content-type", "application/json") + .header("accept", "application/json, text/event-stream") + .json(&initialize) + .send(), + ) + .await + .expect("send initialize request"); let status = response.status(); let response_headers = response.headers().clone(); - let body = response.text().await.expect("read initialize response"); + let body = with_test_timeout( + format!("read MCP initialize response from {endpoint}"), + response.text(), + ) + .await + .expect("read initialize response"); assert_eq!(status, StatusCode::OK, "initialize failed: {body}"); let session_id = response_headers .get("mcp-session-id") @@ -2243,20 +2666,23 @@ impl TestMcpClient { Some("2025-06-18") ); - let notification = apply_headers(client.post(endpoint), &headers) - .header("content-type", "application/json") - .header("accept", "application/json, text/event-stream") - .header("mcp-session-id", &session_id) - .json(&json!({ - "jsonrpc": "2.0", - "method": "notifications/initialized", - })) - .send() - .await - .expect("send initialized notification"); - assert_eq!(notification.status(), StatusCode::ACCEPTED); - - Self { + let notification = with_test_timeout( + format!("MCP initialized notification to {endpoint}"), + apply_headers(client.post(endpoint), &headers) + .header("content-type", "application/json") + .header("accept", "application/json, text/event-stream") + .header("mcp-session-id", &session_id) + .json(&json!({ + "jsonrpc": "2.0", + "method": "notifications/initialized", + })) + .send(), + ) + .await + .expect("send initialized notification"); + assert_eq!(notification.status(), StatusCode::ACCEPTED); + + Self { client, endpoint: endpoint.to_string(), session_id, @@ -2336,14 +2762,20 @@ impl TestMcpClient { "method": method, "params": params, })) - .send() + .send(); + let response = with_test_timeout( + format!("MCP request {method} to {}", self.endpoint), + response, + ) .await .unwrap_or_else(|err| panic!("send MCP request {method}: {err}")); let status = response.status(); - let body = response - .text() - .await - .unwrap_or_else(|err| panic!("read MCP response for {method}: {err}")); + let body = with_test_timeout( + format!("read MCP response for {method} from {}", self.endpoint), + response.text(), + ) + .await + .unwrap_or_else(|err| panic!("read MCP response for {method}: {err}")); assert_eq!( status, StatusCode::OK, @@ -2518,10 +2950,19 @@ async fn framework_component_mcp_discovers_compact_surface() { #[tokio::test] async fn framework_component_mcp_matches_http_surface() { - let http = FrameworkMcpHarness::start(true).await; - let mcp_harness = FrameworkMcpHarness::start(true).await; - let mut mcp = mcp_harness.connect().await; - let mut same_state_mcp = http.connect().await; + let http = with_test_timeout( + "start framework HTTP harness", + FrameworkMcpHarness::start(true), + ) + .await; + let mcp_harness = with_test_timeout( + "start framework MCP harness", + FrameworkMcpHarness::start(true), + ) + .await; + let mut mcp = with_test_timeout("connect framework MCP client", mcp_harness.connect()).await; + let mut same_state_mcp = + with_test_timeout("connect same-state MCP client", http.connect()).await; let http_templates: TemplateListResponse = http.get_json("/v1/templates").await; let mcp_templates: Value = mcp @@ -2654,6 +3095,244 @@ async fn framework_component_mcp_matches_http_surface() { ); } +#[tokio::test] +async fn framework_component_rejects_stale_nonlocal_controller_delivery() { + let (_dir, state, state_path, record) = compile_framework_binding_control_state().await; + let app = test_control_state_app(&_dir, state.clone(), state_path); + let mut controller_plan = app.controller_plan.as_ref().clone(); + controller_plan.site_id = "wrong-site".to_string(); + let controller_app = SiteControllerApp { + control: ControlStateApp { + control_state: app.control_state.clone(), + client: app.client.clone(), + state_path: app.state_path.clone(), + run_root: app.run_root.clone(), + state_root: app.state_root.clone(), + mesh_scope: app.mesh_scope.clone(), + control_state_auth_token: app.control_state_auth_token.clone(), + controller_plan: Arc::new(controller_plan), + authority_locks: app.authority_locks.clone(), + runtime: app.runtime.clone(), + }, + router_auth_token: Arc::::from("test-router-auth"), + }; + + let err = match super::site_controller::execute_site_controller_framework_inspect( + &controller_app, + &record, + &state, + FrameworkComponentInspectRequest::ListTemplates, + ) + .await + { + Ok(_) => panic!("stale controller delivery should be rejected"), + Err(err) => err, + }; + assert_eq!(err.0.code, ProtocolErrorCode::ControlStateUnavailable); + assert!( + err.0 + .message + .contains("router framework route overlays are stale"), + "unexpected error: {}", + err.0.message + ); +} + +#[tokio::test] +async fn framework_component_cross_site_routes_forward_through_site_routers() { + let (dir, mut state, _state_path, _) = compile_framework_binding_control_state().await; + state.placement.offered_sites = BTreeMap::from([ + ( + "authority".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + ), + ( + "consumer".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + ), + ]); + state.placement.defaults = PlacementDefaults { + path: Some("authority".to_string()), + ..PlacementDefaults::default() + }; + state.placement.placement_components = BTreeMap::from([ + ("/".to_string(), "authority".to_string()), + ("/admin".to_string(), "consumer".to_string()), + ]); + state.placement.assignments = state.placement.placement_components.clone(); + refresh_capability_instances(&mut state).expect("framework routes should refresh"); + let state_path = dir.path().join("control-state.json"); + write_control_state(&state_path, &state).expect("state should write"); + let app = test_control_state_app(&dir, state, state_path); + let router_public_key_b64 = base64::engine::general_purpose::STANDARD.encode([9u8; 32]); + let authority_router = Router::new().route( + "/v1/controller/router-identity", + axum::routing::get({ + let router_public_key_b64 = router_public_key_b64.clone(); + move || { + let router_public_key_b64 = router_public_key_b64.clone(); + async move { + Json(RouterIdentityResponse { + id: "/site/authority/router".to_string(), + public_key_b64: router_public_key_b64, + }) + } + } + }), + ); + let consumer_router = Router::new().route( + "/v1/controller/router-identity", + axum::routing::get({ + let router_public_key_b64 = router_public_key_b64.clone(); + move || { + let router_public_key_b64 = router_public_key_b64.clone(); + async move { + Json(RouterIdentityResponse { + id: "/site/consumer/router".to_string(), + public_key_b64: router_public_key_b64, + }) + } + } + }), + ); + let (authority_base_url, _authority_handle) = spawn_test_router(authority_router).await; + let (consumer_base_url, _consumer_handle) = spawn_test_router(consumer_router).await; + + let mut authority_plan = app.controller_plan.as_ref().clone(); + authority_plan.peer_site_router_urls = + BTreeMap::from([("consumer".to_string(), consumer_base_url)]); + let authority_app = ControlStateApp { + controller_plan: Arc::new(authority_plan), + ..app.clone() + }; + + let mut consumer_plan = app.controller_plan.as_ref().clone(); + consumer_plan.site_id = "consumer".to_string(); + consumer_plan.router_identity_id = "/site/consumer/router".to_string(); + consumer_plan.peer_site_router_urls = + BTreeMap::from([("authority".to_string(), authority_base_url)]); + consumer_plan.peer_router_mesh_addrs = + BTreeMap::from([("authority".to_string(), "127.0.0.1:24000".to_string())]); + let consumer_app = ControlStateApp { + controller_plan: Arc::new(consumer_plan), + ..app.clone() + }; + + let authority_overlay = framework_route_overlay_payload(&authority_app) + .await + .expect("authority framework routes should materialize") + .expect("authority site should get a framework route overlay"); + let consumer_overlay = framework_route_overlay_payload(&consumer_app) + .await + .expect("consumer framework routes should materialize") + .expect("consumer site should get a framework route overlay"); + + assert!( + authority_overlay + .peers + .iter() + .any(|peer| peer.id == "/site/consumer/router"), + "authority router should accept framework traffic from the consumer router", + ); + assert!( + consumer_overlay + .peers + .iter() + .any(|peer| peer.id == "/site/authority/router"), + "consumer router should forward framework traffic to the authority router", + ); + + assert!( + consumer_overlay.inbound_routes.iter().any(|route| matches!( + &route.target, + InboundTarget::MeshForward { + peer_id, + peer_addr, + route_id, + capability, + } if peer_id == "/site/authority/router" + && peer_addr == "127.0.0.1:24000" + && route_id == &route.route_id + && capability == &route.capability + )), + "cross-site framework requests must enter the consumer router and cross the router mesh", + ); + + assert!( + authority_overlay + .inbound_routes + .iter() + .any(|route| matches!( + &route.target, + InboundTarget::External { url_env, optional } + if url_env == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV && !optional + )), + "the authority router should hand framework requests to its local site controller only \ + after the router hop", + ); +} + +#[tokio::test] +async fn prepare_child_on_site_rejects_nonlocal_site_plan() { + let dir = TempDir::new().expect("temp dir"); + let root_path = dir.path().join("root.json5"); + let worker_path = dir.path().join("worker.json5"); + write_file( + &worker_path, + r#" + { + manifest_version: "0.3.0", + program: { + path: "/bin/echo", + args: ["worker"], + network: { endpoints: [{ name: "http", port: 8080 }] } + } + } + "#, + ); + write_file( + &root_path, + &format!( + r#" + {{ + manifest_version: "0.3.0", + program: {{ path: "/bin/echo", args: ["root"] }}, + slots: {{ + realm: {{ kind: "component", optional: true }} + }}, + child_templates: {{ + worker: {{ + manifest: "{worker}" + }} + }} + }} + "#, + worker = file_url(&worker_path), + ), + ); + let state = compile_control_state(&root_path).await; + let state_path = dir.path().join("control-state.json"); + write_control_state(&state_path, &state).expect("state should write"); + let app = test_control_state_app(&dir, state, state_path); + let mut child = empty_live_child(1, "remote", 1, ChildState::CreatePrepared); + child.assignments = BTreeMap::from([("/remote".to_string(), "other-site".to_string())]); + let err = prepare_child_on_site(&app, &app.control_state.lock().await.clone(), &child) + .await + .expect_err("nonlocal children should be rejected"); + assert_eq!(err.code, ProtocolErrorCode::PrepareFailed); + assert!( + err.message.contains("only creates local children"), + "unexpected error: {}", + err.message + ); +} + struct DynamicCapsMcpHarness { _dir: TempDir, client: Client, @@ -2899,6 +3578,54 @@ async fn install_dynamic_caps_origin_fixture(app: &ControlStateApp) -> tokio::ta overlay_handle } +#[tokio::test] +async fn dynamic_caps_held_list_ignores_unrouted_offered_sites() { + let dir = TempDir::new().expect("temp dir"); + let mut state = compile_dynamic_caps_binding_state().await; + state.placement.offered_sites.insert( + "compose_local".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + }, + ); + state.placement.offered_sites.insert( + "vm_local".to_string(), + SiteDefinition { + kind: SiteKind::Vm, + context: None, + }, + ); + let state_path = dir.path().join("control-state.json"); + write_control_state(&state_path, &state).expect("state should write"); + let controller_app = SiteControllerApp { + control: test_control_state_app(&dir, state, state_path), + router_auth_token: Arc::::from("test-router-auth"), + }; + + let response = super::site_controller::execute_site_controller_dynamic_caps_inspect( + &controller_app, + super::control_state_api::DynamicCapsInspectRequest::HeldList( + dynamic_caps::ControlDynamicHeldListRequest { + holder_component_id: "components./alice".to_string(), + }, + ), + false, + ) + .await + .expect("held list should stay local when no peer controller routes exist"); + + let super::control_state_api::DynamicCapsInspectResponse::HeldList(held) = response else { + panic!("held list request should return a held list response"); + }; + assert!( + held.held + .iter() + .any(|entry| entry.entry_kind == HeldEntryKind::RootAuthority), + "local held roots should still be returned", + ); +} + #[tokio::test] async fn dynamic_caps_mcp_discovers_compact_surface() { let harness = DynamicCapsMcpHarness::start().await; @@ -3176,290 +3903,6 @@ async fn dynamic_caps_mcp_matches_http_surface() { ); } -#[derive(Clone)] -struct MockSiteControllerState { - expected_auth: String, - rollback_status: StatusCode, -} - -async fn mock_site_controller_status( - State(state): State, - headers: HeaderMap, -) -> StatusCode { - assert_eq!( - headers - .get(FRAMEWORK_AUTH_HEADER) - .and_then(|value| value.to_str().ok()), - Some(state.expected_auth.as_str()), - "site controller requests should authenticate with the control token", - ); - StatusCode::NO_CONTENT -} - -async fn mock_site_controller_rollback_status( - State(state): State, - headers: HeaderMap, -) -> StatusCode { - assert_eq!( - headers - .get(FRAMEWORK_AUTH_HEADER) - .and_then(|value| value.to_str().ok()), - Some(state.expected_auth.as_str()), - "site controller requests should authenticate with the control token", - ); - state.rollback_status -} - -async fn install_mock_site_controllers( - app: &ControlStateApp, - rollback_status: StatusCode, -) -> (ControlStateApp, Vec>) { - let offered_sites = { - let state = app.control_state.lock().await; - state - .placement - .offered_sites - .iter() - .map(|(site_id, site)| (site_id.clone(), site.kind)) - .collect::>() - }; - let mut peer_controllers = BTreeMap::new(); - let mut handles = Vec::with_capacity(offered_sites.len()); - for (site_id, site_kind) in offered_sites { - let state = MockSiteControllerState { - expected_auth: app.control_state_auth_token.to_string(), - rollback_status, - }; - let router = Router::new() - .route( - "/v1/controller/site/children/{child_id}/prepare", - post(mock_site_controller_status), - ) - .route( - "/v1/controller/site/children/{child_id}/rollback", - post(mock_site_controller_rollback_status), - ) - .route( - "/v1/controller/site/children/{child_id}/publish", - post(mock_site_controller_status), - ) - .route( - "/v1/controller/site/children/{child_id}/destroy", - post(mock_site_controller_status), - ) - .with_state(state); - let (authority_url, handle) = spawn_test_router(router).await; - peer_controllers.insert( - site_id.clone(), - SiteControllerPeerPlan { - site_id, - kind: site_kind, - authority_url, - }, - ); - handles.push(handle); - } - (with_peer_site_controllers(app, peer_controllers), handles) -} - -async fn install_success_site_controllers( - app: &ControlStateApp, -) -> (ControlStateApp, Vec>) { - install_mock_site_controllers(app, StatusCode::NO_CONTENT).await -} - -async fn install_failing_rollback_site_controllers( - app: &ControlStateApp, -) -> (ControlStateApp, Vec>) { - install_mock_site_controllers(app, StatusCode::INTERNAL_SERVER_ERROR).await -} - -fn with_peer_site_controllers( - app: &ControlStateApp, - peer_controllers: BTreeMap, -) -> ControlStateApp { - let mut controller_plan = app.controller_plan.as_ref().clone(); - controller_plan.site_id = "__test_controller__".to_string(); - controller_plan.peer_controllers = peer_controllers.clone(); - ControlStateApp { - control_state: app.control_state.clone(), - client: app.client.clone(), - state_path: app.state_path.clone(), - run_root: app.run_root.clone(), - state_root: app.state_root.clone(), - mesh_scope: app.mesh_scope.clone(), - control_state_auth_token: app.control_state_auth_token.clone(), - controller_plan: Arc::new(controller_plan), - peer_controllers: Arc::new(peer_controllers), - authority_locks: app.authority_locks.clone(), - runtime: app.runtime.clone(), - } -} - -async fn install_barrier_destroy_site_controllers( - app: &ControlStateApp, -) -> ( - ControlStateApp, - Vec>, - tokio::sync::mpsc::UnboundedReceiver, - Arc, -) { - let offered_sites = { - let state = app.control_state.lock().await; - state - .placement - .offered_sites - .iter() - .map(|(site_id, site)| (site_id.clone(), site.kind)) - .collect::>() - }; - let barrier = Arc::new(tokio::sync::Barrier::new(offered_sites.len() + 1)); - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let mut peer_controllers = BTreeMap::new(); - let mut handles = Vec::with_capacity(offered_sites.len()); - for (site_id, site_kind) in offered_sites { - let state = MockSiteControllerState { - expected_auth: app.control_state_auth_token.to_string(), - rollback_status: StatusCode::NO_CONTENT, - }; - let start_tx = tx.clone(); - let destroy_barrier = barrier.clone(); - let site_id_for_destroy = site_id.clone(); - let router = Router::new() - .route( - "/v1/controller/site/children/{child_id}/prepare", - post(mock_site_controller_status), - ) - .route( - "/v1/controller/site/children/{child_id}/publish", - post(mock_site_controller_status), - ) - .route( - "/v1/controller/site/children/{child_id}/destroy", - post( - move |State(state): State, headers: HeaderMap| { - let start_tx = start_tx.clone(); - let destroy_barrier = destroy_barrier.clone(); - let site_id = site_id_for_destroy.clone(); - async move { - assert_eq!( - headers - .get(FRAMEWORK_AUTH_HEADER) - .and_then(|value| value.to_str().ok()), - Some(state.expected_auth.as_str()), - "site controller requests should authenticate with the control \ - token", - ); - start_tx - .send(site_id) - .expect("destroy start notification should send"); - destroy_barrier.wait().await; - StatusCode::NO_CONTENT - } - }, - ), - ); - let (authority_url, handle) = spawn_test_router(router.with_state(state)).await; - peer_controllers.insert( - site_id.clone(), - SiteControllerPeerPlan { - site_id, - kind: site_kind, - authority_url, - }, - ); - handles.push(handle); - } - ( - with_peer_site_controllers(app, peer_controllers), - handles, - rx, - barrier, - ) -} - -async fn install_barrier_publish_site_controllers( - app: &ControlStateApp, -) -> ( - ControlStateApp, - Vec>, - tokio::sync::mpsc::UnboundedReceiver, - Arc, -) { - let offered_sites = { - let state = app.control_state.lock().await; - state - .placement - .offered_sites - .iter() - .map(|(site_id, site)| (site_id.clone(), site.kind)) - .collect::>() - }; - let barrier = Arc::new(tokio::sync::Barrier::new(offered_sites.len() + 1)); - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let mut peer_controllers = BTreeMap::new(); - let mut handles = Vec::with_capacity(offered_sites.len()); - for (site_id, site_kind) in offered_sites { - let state = MockSiteControllerState { - expected_auth: app.control_state_auth_token.to_string(), - rollback_status: StatusCode::NO_CONTENT, - }; - let start_tx = tx.clone(); - let publish_barrier = barrier.clone(); - let site_id_for_publish = site_id.clone(); - let router = Router::new() - .route( - "/v1/controller/site/children/{child_id}/prepare", - post(mock_site_controller_status), - ) - .route( - "/v1/controller/site/children/{child_id}/publish", - post( - move |State(state): State, headers: HeaderMap| { - let start_tx = start_tx.clone(); - let publish_barrier = publish_barrier.clone(); - let site_id = site_id_for_publish.clone(); - async move { - assert_eq!( - headers - .get(FRAMEWORK_AUTH_HEADER) - .and_then(|value| value.to_str().ok()), - Some(state.expected_auth.as_str()), - "site controller requests should authenticate with the control \ - token", - ); - start_tx - .send(site_id) - .expect("publish start notification should send"); - publish_barrier.wait().await; - StatusCode::NO_CONTENT - } - }, - ), - ) - .route( - "/v1/controller/site/children/{child_id}/destroy", - post(mock_site_controller_status), - ); - let (authority_url, handle) = spawn_test_router(router.with_state(state)).await; - peer_controllers.insert( - site_id.clone(), - SiteControllerPeerPlan { - site_id, - kind: site_kind, - authority_url, - }, - ); - handles.push(handle); - } - ( - with_peer_site_controllers(app, peer_controllers), - handles, - rx, - barrier, - ) -} - #[tokio::test] async fn create_snapshot_and_destroy_exact_child() { let dir = TempDir::new().expect("temp dir"); @@ -3883,8 +4326,13 @@ async fn open_template_replay_uses_admitted_manifest_after_source_mutation() { "replay should restore the child fragment as authoritative semantic state", ); assert!( - !replayed.live_children[0].site_plans.is_empty(), - "replay should rebuild derived site plans from the restored child fragment", + build_local_child_runtime_spec( + &replayed, + &replayed.live_children[0], + &child_runtime_site_id(&replayed.live_children[0]).expect("replayed child site id"), + ) + .is_ok(), + "replay should rebuild local runtime realization from the restored child fragment", ); assert!( list_children(&replayed, replay_root_authority) @@ -4896,10 +5344,7 @@ async fn create_aborts_and_destroys_partially_published_child_when_publish_fails ); assert_eq!(destroy_calls[0].0, 1); assert_eq!( - destroy_calls[0] - .1 - .as_ref() - .map(|site_plan| site_plan.site_id.as_str()), + Some(destroy_calls[0].1.as_str()), Some("direct_local"), "failed publish cleanup must reconcile the child site back to the desired plan", ); @@ -5163,7 +5608,6 @@ async fn concurrent_same_name_creates_serialize_to_one_live_child() { let (dir, state, state_path) = compile_exact_template_control_state().await; let root_authority = state.base_scenario.root; let app = test_control_state_app(&dir, state, state_path); - let (app, controllers) = install_success_site_controllers(&app).await; let request = CreateChildRequest { template: "worker".to_string(), name: "job".to_string(), @@ -5212,322 +5656,25 @@ async fn concurrent_same_name_creates_serialize_to_one_live_child() { 1, "snapshot should remain clean after the same-name race", ); - for controller in controllers { - controller.abort(); - } } #[tokio::test] -async fn destroy_retracted_tears_down_sites_concurrently() { +async fn concurrent_distinct_creates_commit_both_children() { let dir = TempDir::new().expect("temp dir"); let root_path = dir.path().join("root.json5"); + let child_path = dir.path().join("child.json5"); write_file( - &root_path, + &child_path, r#" { manifest_version: "0.3.0", program: { - path: "/bin/sh", - args: ["-c", "sleep 1"] - } - } - "#, - ); - - let placement = PlacementFile { - schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), - version: amber_compiler::run_plan::PLACEMENT_VERSION, - sites: BTreeMap::from([ - ( - "compose_local".to_string(), - SiteDefinition { - kind: SiteKind::Compose, - context: None, - }, - ), - ( - "direct_local".to_string(), - SiteDefinition { - kind: SiteKind::Direct, - context: None, - }, - ), - ]), - defaults: PlacementDefaults { - path: Some("direct_local".to_string()), - image: Some("compose_local".to_string()), - ..PlacementDefaults::default() - }, - components: BTreeMap::new(), - dynamic_capabilities: None, - framework_children: None, - }; - - let state = compile_control_state_with_placement(&root_path, Some(&placement)).await; - let state_path = dir.path().join("control-state.json"); - write_control_state(&state_path, &state).expect("state should write"); - let root_authority = state.base_scenario.root; - let app = test_control_state_app(&dir, state, state_path); - let (app, controllers, mut destroy_starts, barrier) = - install_barrier_destroy_site_controllers(&app).await; - { - let mut state = app.control_state.lock().await; - state.pending_destroys.push(pending_destroy( - 1, - LiveChildRecord { - child_id: 7, - authority_realm_id: root_authority, - name: "job-compose".to_string(), - state: ChildState::DestroyRetracted, - template_name: Some("fixture".to_string()), - selected_manifest_catalog_key: None, - fragment: None, - input_bindings: Vec::new(), - assignments: BTreeMap::new(), - site_plans: vec![ - DynamicSitePlanRecord { - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - router_identity_id: "/site/compose_local/router".to_string(), - component_ids: Vec::new(), - assigned_components: Vec::new(), - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), - }, - DynamicSitePlanRecord { - site_id: "direct_local".to_string(), - kind: SiteKind::Direct, - router_identity_id: "/site/direct_local/router".to_string(), - component_ids: Vec::new(), - assigned_components: Vec::new(), - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), - }, - ], - overlay_ids: Vec::new(), - overlays: Vec::new(), - outputs: BTreeMap::new(), - }, - )); - } - - let destroy = tokio::spawn({ - let app = app.clone(); - async move { continue_destroy_retracted(&app, 7).await } - }); - - let first = tokio::time::timeout(Duration::from_secs(5), destroy_starts.recv()) - .await - .expect("first destroy should start in time") - .expect("first destroy notification should arrive"); - let second = tokio::time::timeout(Duration::from_secs(5), destroy_starts.recv()) - .await - .expect("second destroy should start in time") - .expect("second destroy notification should arrive"); - assert_ne!( - first, second, - "destroy should reach both peer site controllers before either completes" - ); - - barrier.wait().await; - destroy - .await - .expect("destroy task should join") - .expect("destroy should succeed"); - - let recovered = app.control_state.lock().await.clone(); - assert!( - recovered.live_children.is_empty(), - "successful destroy should remove the child after concurrent site teardown", - ); - for controller in controllers { - controller.abort(); - } -} - -#[tokio::test] -async fn create_committed_hidden_publishes_independent_sites_concurrently() { - let dir = TempDir::new().expect("temp dir"); - let root_path = dir.path().join("root.json5"); - let child_path = dir.path().join("child.json5"); - write_file( - &child_path, - r#" - { - manifest_version: "0.3.0", - program: { - path: "/bin/echo", - args: ["child"], - network: { endpoints: [{ name: "out", port: 8080, protocol: "http" }] } - }, - provides: { out: { kind: "http", endpoint: "out" } }, - exports: { out: "provides.out" }, - } - "#, - ); - write_file( - &root_path, - &format!( - r#" - {{ - manifest_version: "0.3.0", - program: {{ - path: "/bin/sh", - args: ["-c", "sleep 1"] - }}, - slots: {{ - realm: {{ kind: "component", optional: true }} - }}, - child_templates: {{ - fixture: {{ manifest: "{child}" }} - }} - }} - "#, - child = file_url(&child_path), - ), - ); - - let placement = PlacementFile { - schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), - version: amber_compiler::run_plan::PLACEMENT_VERSION, - sites: BTreeMap::from([ - ( - "compose_local".to_string(), - SiteDefinition { - kind: SiteKind::Compose, - context: None, - }, - ), - ( - "direct_local".to_string(), - SiteDefinition { - kind: SiteKind::Direct, - context: None, - }, - ), - ]), - defaults: PlacementDefaults { - path: Some("direct_local".to_string()), - image: Some("compose_local".to_string()), - ..PlacementDefaults::default() - }, - components: BTreeMap::new(), - dynamic_capabilities: None, - framework_children: None, - }; - - let mut state = compile_control_state_with_placement(&root_path, Some(&placement)).await; - let state_path = dir.path().join("control-state.json"); - let root_authority = state.base_scenario.root; - let mut child = prepare_child_record( - &mut state, - root_authority, - &CreateChildRequest { - template: "fixture".to_string(), - name: "job-compose".to_string(), - manifest: None, - config: BTreeMap::new(), - bindings: BTreeMap::new(), - }, - ) - .await - .expect("child should plan successfully"); - let child_id = child.child_id; - child.state = ChildState::CreateCommittedHidden; - child.site_plans = vec![ - DynamicSitePlanRecord { - site_id: "compose_local".to_string(), - kind: SiteKind::Compose, - router_identity_id: "/site/compose_local/router".to_string(), - component_ids: Vec::new(), - assigned_components: Vec::new(), - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), - }, - DynamicSitePlanRecord { - site_id: "direct_local".to_string(), - kind: SiteKind::Direct, - router_identity_id: "/site/direct_local/router".to_string(), - component_ids: Vec::new(), - assigned_components: Vec::new(), - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), - }, - ]; - write_control_state(&state_path, &state).expect("state should write"); - let app = test_control_state_app(&dir, state, state_path); - let (app, controllers, mut publish_starts, barrier) = - install_barrier_publish_site_controllers(&app).await; - { - let mut state = app.control_state.lock().await; - state.pending_creates.push(pending_create(1, child)); - } - - let publish = tokio::spawn({ - let app = app.clone(); - async move { continue_create_committed_hidden(&app, child_id).await } - }); - - let first = tokio::time::timeout(Duration::from_secs(5), publish_starts.recv()) - .await - .expect("first publish should start in time") - .expect("first publish notification should arrive"); - let second = tokio::time::timeout(Duration::from_secs(5), publish_starts.recv()) - .await - .expect("second publish should start in time") - .expect("second publish notification should arrive"); - assert_ne!( - first, second, - "create should reach both independent site controllers before either completes" - ); - - barrier.wait().await; - publish - .await - .expect("publish task should join") - .expect("publish should succeed"); - - let recovered = app.control_state.lock().await.clone(); - let child = recovered - .live_children - .iter() - .find(|child| child.child_id == child_id) - .expect("child should remain present"); - assert_eq!( - child.state, - ChildState::Live, - "successful concurrent site publication should promote the child to live", - ); - for controller in controllers { - controller.abort(); - } -} - -#[tokio::test] -async fn concurrent_distinct_creates_commit_both_children() { - let dir = TempDir::new().expect("temp dir"); - let root_path = dir.path().join("root.json5"); - let child_path = dir.path().join("child.json5"); - write_file( - &child_path, - r#" - { - manifest_version: "0.3.0", - program: { - image: "busybox:1.36.1", - entrypoint: ["/bin/sh", "-c", "sleep 1"], - network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] } - }, - provides: { http: { kind: "http", endpoint: "http" } }, - exports: { http: "provides.http" }, + image: "busybox:1.36.1", + entrypoint: ["/bin/sh", "-c", "sleep 1"], + network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] } + }, + provides: { http: { kind: "http", endpoint: "http" } }, + exports: { http: "provides.http" }, } "#, ); @@ -5571,7 +5718,6 @@ async fn concurrent_distinct_creates_commit_both_children() { write_control_state(&state_path, &state).expect("state should write"); let root_authority = state.base_scenario.root; let app = test_control_state_app(&dir, state, state_path); - let (app, controllers) = install_success_site_controllers(&app).await; let (left, right) = tokio::join!( execute_create_child( @@ -5632,9 +5778,6 @@ async fn concurrent_distinct_creates_commit_both_children() { .any(|component| component.moniker == "/job-b"), "snapshot should contain the second child", ); - for controller in controllers { - controller.abort(); - } } #[tokio::test] @@ -5727,7 +5870,7 @@ async fn prepare_child_record_uses_frozen_dynamic_placement_assignments() { } #[tokio::test] -async fn prepare_child_record_preserves_cross_backend_matrix_assignments() { +async fn prepare_child_record_rejects_cross_site_dynamic_fragments() { let dir = TempDir::new().expect("temp dir"); let root_path = dir.path().join("root.json5"); let child_path = dir.path().join("child-compose.json5"); @@ -5935,7 +6078,7 @@ async fn prepare_child_record_preserves_cross_backend_matrix_assignments() { let mut state = compile_control_state_with_placement(&root_path, Some(&placement)).await; let root_authority = state.base_scenario.root; - let child = prepare_child_record( + let err = prepare_child_record( &mut state, root_authority, &CreateChildRequest { @@ -5947,88 +6090,12 @@ async fn prepare_child_record_preserves_cross_backend_matrix_assignments() { }, ) .await - .expect("matrix child should plan successfully"); - - assert_eq!( - child - .assignments - .get("/job-compose/root") - .map(String::as_str), - Some("compose_local"), - ); - assert_eq!( - child - .assignments - .get("/job-compose/kind_helper") - .map(String::as_str), - Some("kind_local"), - ); - assert_eq!( - child - .assignments - .get("/job-compose/direct_helper") - .map(String::as_str), - Some("direct_local"), - ); - assert_eq!( - child - .assignments - .get("/job-compose/vm_helper/root") - .map(String::as_str), - Some("vm_local"), - ); - assert_eq!( - child - .site_plans - .iter() - .map(|site_plan| site_plan.site_id.as_str()) - .collect::>(), - BTreeSet::from(["compose_local", "direct_local", "kind_local", "vm_local"]), - "cross-backend child planning should retain all expected site slices", - ); - let proxy_exports_by_site = child - .site_plans - .iter() - .map(|site_plan| { - ( - site_plan.site_id.as_str(), - site_plan - .proxy_exports - .keys() - .map(String::as_str) - .collect::>(), - ) - }) - .collect::>(); - assert_eq!( - proxy_exports_by_site.get("compose_local"), - Some(&BTreeSet::from(["http"])), - "compose site should own the dynamic child root export", - ); - for (site_id, public_export) in [ - ("kind_local", "kind_http"), - ("direct_local", "direct_http"), - ("vm_local", "vm_http"), - ] { - let exports = proxy_exports_by_site - .get(site_id) - .unwrap_or_else(|| panic!("missing proxy export set for {site_id}")); - assert!( - exports.contains(public_export), - "{site_id} should keep its public helper export", - ); - assert!( - exports.iter().any(|name| name.starts_with("amber_export_")), - "{site_id} should also publish its internal routed link export", - ); - } + .expect_err("site controllers must reject dynamic children that span multiple sites"); + assert_eq!(err.code, ProtocolErrorCode::PlacementUnsatisfied); assert!( - child - .site_plans - .iter() - .all(|site_plan| site_plan.routed_inputs.is_empty()), - "bindings that stay inside the created fragment must remain intra-fragment wiring, not \ - site-router routed inputs", + err.message.contains("spans multiple sites"), + "cross-site dynamic child placement should be rejected explicitly, got: {}", + err.message ); } @@ -6631,35 +6698,13 @@ async fn recover_control_state_surfaces_create_prepared_rollback_failures() { let mut state = compile_control_state_with_placement(&root_path, Some(&placement)).await; let state_path = dir.path().join("control-state.json"); let root_authority = state.base_scenario.root; - let app = test_control_state_app(&dir, state.clone(), state_path.clone()); - let (app, controllers) = install_failing_rollback_site_controllers(&app).await; + let app = with_runtime( + &test_control_state_app(&dir, state.clone(), state_path.clone()), + Arc::new(FailingRollbackRuntime), + ); state.pending_creates.push(pending_create( 1, - LiveChildRecord { - child_id: 1, - authority_realm_id: root_authority, - name: "prepared".to_string(), - state: ChildState::CreatePrepared, - template_name: Some("worker".to_string()), - selected_manifest_catalog_key: None, - fragment: None, - input_bindings: Vec::new(), - assignments: BTreeMap::new(), - site_plans: vec![DynamicSitePlanRecord { - site_id: "direct_local".to_string(), - kind: SiteKind::Direct, - router_identity_id: "/site/direct_local/router".to_string(), - component_ids: Vec::new(), - assigned_components: Vec::new(), - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), - }], - overlay_ids: Vec::new(), - overlays: Vec::new(), - outputs: BTreeMap::new(), - }, + empty_live_child(root_authority, "prepared", 1, ChildState::CreatePrepared), )); write_control_state(&state_path, &state).expect("state should write"); *app.control_state.lock().await = state; @@ -6683,9 +6728,6 @@ async fn recover_control_state_surfaces_create_prepared_rollback_failures() { recovered.journal.is_empty(), "failed rollback must not pretend the child was aborted" ); - for controller in controllers { - controller.abort(); - } } #[tokio::test] @@ -6706,7 +6748,6 @@ async fn recover_control_state_promotes_create_committed_hidden_children_to_live .await .expect("child should plan successfully"); child.state = ChildState::CreateCommittedHidden; - child.site_plans.clear(); state.pending_creates.push(pending_create(1, child)); write_control_state(&state_path, &state).expect("state should write"); let app = test_control_state_app(&dir, state, state_path); @@ -6778,34 +6819,27 @@ async fn recover_control_state_does_not_republish_live_children() { fragment: None, input_bindings: Vec::new(), assignments: BTreeMap::new(), - site_plans: vec![DynamicSitePlanRecord { - site_id: "direct_local".to_string(), - kind: SiteKind::Direct, - router_identity_id: "/site/direct_local/router".to_string(), - component_ids: Vec::new(), - assigned_components: Vec::new(), - artifact_files: BTreeMap::new(), - desired_artifact_files: BTreeMap::new(), - proxy_exports: BTreeMap::new(), - routed_inputs: Vec::new(), - }], overlay_ids: Vec::new(), overlays: Vec::new(), outputs: BTreeMap::new(), }); write_control_state(&state_path, &state).expect("state should write"); - let app = test_control_state_app(&dir, state, state_path); - let (_app, controllers, mut publish_starts, _barrier) = - install_barrier_publish_site_controllers(&app).await; + let runtime = Arc::new(RecordingPublishRuntime::default()); + let app = with_runtime( + &test_control_state_app(&dir, state, state_path), + runtime.clone(), + ); recover_control_state(&app) .await .expect("recovery should leave live children alone"); assert!( - tokio::time::timeout(Duration::from_millis(200), publish_starts.recv()) - .await - .is_err(), + runtime + .publish_calls + .lock() + .expect("publish call log mutex should lock") + .is_empty(), "live recovery should not call publish again", ); let recovered = app.control_state.lock().await.clone(); @@ -6816,9 +6850,6 @@ async fn recover_control_state_does_not_republish_live_children() { recovered.journal.is_empty(), "live recovery should not append synthetic journal entries", ); - for controller in controllers { - controller.abort(); - } } #[tokio::test] @@ -7463,3 +7494,301 @@ async fn dynamic_capabilities_inspect_ref_rejects_unsupported_token_versions() { assert_eq!(err.code, ProtocolErrorCode::MalformedRef); assert!(err.message.contains("unsupported")); } + +#[test] +fn compose_site_controller_is_injected_as_a_compose_service() { + let temp = TempDir::new().expect("tempdir should create"); + let artifact_root = temp.path().join("artifact"); + let site_state_root = temp.path().join("state").join("compose-site"); + fs::create_dir_all(&artifact_root).expect("artifact root should create"); + fs::create_dir_all(&site_state_root).expect("site state root should create"); + fs::write( + artifact_root.join("compose.yaml"), + r#" +services: + amber-router: + image: ghcr.io/rdi-foundation/amber-router:test + volumes: + - amber-router-control:/amber/control + amber-router-control-init: + image: busybox + amber-provisioner: + image: ghcr.io/rdi-foundation/amber-provisioner:test +networks: + amber_mesh: {} +volumes: + amber-router-control: {} +"#, + ) + .expect("compose yaml should write"); + + let plan = write_site_controller_plan( + &site_controller_plan_path(&site_state_root), + "test-run", + "test-mesh", + "compose-site", + SiteKind::Compose, + SocketAddr::from(([0, 0, 0, 0], SITE_CONTROLLER_PORT)), + &format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{SITE_CONTROLLER_PORT}"), + "/site/compose-site/router", + &BTreeMap::new(), + &BTreeMap::new(), + Some("unix:///amber/control/router-control.sock"), + Some("127.0.0.1:24000"), + &site_state_root.join("site-controller-state.json"), + temp.path(), + &temp.path().join("state"), + &site_state_root, + &artifact_root, + "test-auth", + "test-verify-key", + None, + None, + Some(24000), + Some("amber_test_compose"), + None, + None, + None, + &BTreeMap::new(), + ) + .expect("compose site controller plan should write"); + + inject_compose_site_controller( + &artifact_root, + &plan, + &site_controller_plan_path(&site_state_root), + "ghcr.io/rdi-foundation/amber-site-controller:test", + ) + .expect("compose controller should inject"); + + let document: serde_yaml::Value = serde_yaml::from_str( + &fs::read_to_string(artifact_root.join("compose.yaml")).expect("compose yaml should read"), + ) + .expect("compose yaml should parse"); + let service = document["services"][SITE_CONTROLLER_SERVICE_NAME] + .as_mapping() + .expect("controller service should exist"); + assert_eq!( + service + .get(serde_yaml::Value::String("image".to_string())) + .and_then(serde_yaml::Value::as_str), + Some("ghcr.io/rdi-foundation/amber-site-controller:test") + ); + let command = service + .get(serde_yaml::Value::String("command".to_string())) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller service should have a command"); + let plan_path = site_controller_plan_path(&site_state_root) + .display() + .to_string(); + assert_eq!(command[0].as_str(), Some("--plan")); + assert_eq!(command[1].as_str(), Some(plan_path.as_str())); + let volumes = service + .get(serde_yaml::Value::String("volumes".to_string())) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller service should mount volumes"); + assert!(volumes.iter().any(|value| { + value.as_str() + == Some(&format!( + "{}:{}", + temp.path().display(), + temp.path().display() + )) + })); + assert!( + volumes + .iter() + .any(|value| value.as_str() == Some("amber-router-control:/amber/control")) + ); + assert!( + volumes + .iter() + .any(|value| value.as_str() == Some("/var/run/docker.sock:/var/run/docker.sock")) + ); +} + +#[test] +fn kubernetes_site_controller_resources_are_injected_into_the_artifact() { + let temp = TempDir::new().expect("tempdir should create"); + let artifact_root = temp.path().join("artifact"); + let site_state_root = temp.path().join("state").join("kube-site"); + fs::create_dir_all(artifact_root.join("05-networkpolicies")) + .expect("network policies dir should create"); + fs::create_dir_all(site_state_root.clone()).expect("site state root should create"); + fs::write( + artifact_root.join("kustomization.yaml"), + "resources:\n - 05-networkpolicies/amber-router-netpol.yaml\n", + ) + .expect("kustomization should write"); + fs::write( + artifact_root.join("05-networkpolicies/amber-router-netpol.yaml"), + r#" +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: amber-router +spec: + ingress: [] +"#, + ) + .expect("router netpol should write"); + fs::write( + site_state_root.join("site-controller-state.json"), + "{\"schema\":\"amber.test\",\"version\":1}", + ) + .expect("controller state should write"); + fs::write( + site_state_root.join("desired-links.json"), + "{\"schema\":\"amber.test\",\"version\":1}", + ) + .expect("desired links should write"); + + let plan = write_site_controller_plan( + &site_controller_plan_path(&site_state_root), + "test-run", + "test-mesh", + "kube-site", + SiteKind::Kubernetes, + SocketAddr::from(([0, 0, 0, 0], SITE_CONTROLLER_PORT)), + &format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{SITE_CONTROLLER_PORT}"), + "/site/kube-site/router", + &BTreeMap::new(), + &BTreeMap::new(), + Some("amber-router:24100"), + Some("127.0.0.1:24000"), + &site_state_root.join("site-controller-state.json"), + temp.path(), + &temp.path().join("state"), + &site_state_root, + &artifact_root, + "test-auth", + "test-verify-key", + None, + None, + Some(24000), + None, + Some("amber-test-kube-site"), + None, + None, + &BTreeMap::new(), + ) + .expect("kubernetes site controller plan should write"); + + inject_kubernetes_site_controller( + &artifact_root, + &plan, + "ghcr.io/rdi-foundation/amber-site-controller:test", + ) + .expect("kubernetes controller should inject"); + + let kustomization = fs::read_to_string(artifact_root.join("kustomization.yaml")) + .expect("kustomization should read"); + assert!(kustomization.contains("01-configmaps/amber-site-controller-seed.yaml")); + assert!(kustomization.contains("03-deployments/amber-site-controller.yaml")); + assert!(kustomization.contains("04-services/amber-site-controller.yaml")); + + let deployment = + fs::read_to_string(artifact_root.join("03-deployments/amber-site-controller.yaml")) + .expect("deployment should read"); + assert!(deployment.contains("amber-site-controller")); + assert!(deployment.contains("ghcr.io/rdi-foundation/amber-site-controller:test")); + assert!(deployment.contains("/amber/site/state/site-controller-plan.json")); + + let seed = + fs::read_to_string(artifact_root.join("01-configmaps/amber-site-controller-seed.yaml")) + .expect("seed configmap should read"); + assert!(seed.contains("site-controller-plan.json")); + assert!(seed.contains("artifact.tar.b64")); + assert!(seed.contains("http://amber-site-controller:4100")); + + let router_netpol = + fs::read_to_string(artifact_root.join("05-networkpolicies/amber-router-netpol.yaml")) + .expect("router netpol should read"); + assert!(router_netpol.contains("amber-site-controller")); + assert!(router_netpol.contains("24100")); +} + +#[test] +fn local_site_manager_state_uses_controller_plan_when_host_state_is_absent() { + let temp = TempDir::new().expect("tempdir should create"); + let state_root = temp.path().join("state"); + let site_state_root = state_root.join("compose-site"); + fs::create_dir_all(&site_state_root).expect("site state root should create"); + let state = FrameworkControlState { + schema: CONTROL_STATE_SCHEMA.to_string(), + version: CONTROL_STATE_VERSION, + run_id: "test-run".to_string(), + base_scenario: ScenarioIr { + schema: amber_scenario::SCENARIO_IR_SCHEMA.to_string(), + version: amber_scenario::SCENARIO_IR_VERSION, + root: 0, + components: Vec::new(), + bindings: Vec::new(), + exports: Vec::new(), + manifest_catalog: BTreeMap::new(), + }, + run_links: Vec::new(), + placement: FrozenPlacementState { + offered_sites: BTreeMap::from([( + "compose-site".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + }, + )]), + defaults: PlacementDefaults::default(), + standby_sites: Vec::new(), + initial_active_sites: vec!["compose-site".to_string()], + dynamic_enabled_sites: vec!["compose-site".to_string()], + control_only_sites: Vec::new(), + active_site_capabilities: BTreeMap::new(), + placement_components: BTreeMap::new(), + assignments: BTreeMap::new(), + }, + generation: 0, + next_child_id: 1, + next_tx_id: 0, + id_stride: 1, + next_component_id: 0, + capability_instances: BTreeMap::new(), + journal: Vec::new(), + dynamic_capability_signing_seed_b64: mesh_dynamic_caps::signing_seed_b64( + &mesh_dynamic_caps::signing_key_from_seed( + mesh_dynamic_caps::generate_dynamic_capability_signing_seed(), + ), + ), + next_dynamic_capability_grant_id: 0, + dynamic_capability_grants: BTreeMap::new(), + dynamic_capability_journal: Vec::new(), + live_children: Vec::new(), + pending_creates: Vec::new(), + pending_destroys: Vec::new(), + }; + let state_path = site_state_root.join("site-controller-state.json"); + write_json(&state_path, &state).expect("state should write"); + let mut app = test_control_state_app(&temp, state, state_path); + fs::remove_file(site_state_path(&temp.path().join("state"), "compose-site")) + .expect("host manager state should be removed for fallback test"); + let controller_plan = Arc::make_mut(&mut app.controller_plan); + controller_plan.kind = SiteKind::Compose; + controller_plan.compose_project = Some("amber_test_compose".to_string()); + controller_plan.local_router_control = + Some("unix:///amber/control/router-control.sock".to_string()); + controller_plan.published_router_mesh_addr = Some("127.0.0.1:24000".to_string()); + controller_plan.authority_url = + format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{SITE_CONTROLLER_PORT}"); + + let state = load_site_manager_state(&app, "compose-site") + .expect("local controller should synthesize site metadata from its own plan"); + assert_eq!(state.status, "running"); + assert_eq!( + state.router_control.as_deref(), + Some("unix:///amber/control/router-control.sock") + ); + assert_eq!(state.router_mesh_addr.as_deref(), Some("127.0.0.1:24000")); + let authority_url = format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{SITE_CONTROLLER_PORT}"); + assert_eq!( + state.site_controller_url.as_deref(), + Some(authority_url.as_str()) + ); +} From c9bf4626be8b19d5b2c0e882ba3654ca4387c589 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Mon, 13 Apr 2026 15:30:27 +0800 Subject: [PATCH 03/25] Fix tests --- Cargo.lock | 1 + cli/Cargo.toml | 1 + cli/src/mixed_run.rs | 45 +- cli/src/mixed_run/launch_bundle.rs | 33 +- cli/src/mixed_run/supervisor.rs | 124 +- cli/src/mixed_run/tests.rs | 193 ++++ cli/tests/mixed_run/support.rs | 111 +- compiler/src/targets/mesh/mesh_config.rs | 132 ++- docker/images.json | 2 +- manager/src/runtime.rs | 25 +- runtime/router/src/control.rs | 2 + runtime/router/src/dynamic_caps.rs | 1 + runtime/router/src/lib.rs | 27 +- runtime/router/src/tests.rs | 220 ++++ runtime/router/src/transport.rs | 2 + .../site-controller/src/default_runtime.rs | 4 +- .../src/default_runtime/child_runtime.rs | 94 +- .../src/default_runtime/compose_controller.rs | 11 + .../src/default_runtime/dynamic_routes.rs | 21 +- .../src/default_runtime/site_artifacts.rs | 344 +++++- runtime/site-controller/src/dynamic_caps.rs | 19 + runtime/site-controller/src/http.rs | 4 - runtime/site-controller/src/orchestration.rs | 48 +- runtime/site-controller/src/planner.rs | 1 + .../site-controller/src/site_controller.rs | 173 ++- runtime/site-controller/src/state.rs | 108 +- runtime/site-controller/src/tests.rs | 1002 ++++++++++++++++- 27 files changed, 2574 insertions(+), 174 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd7cdba6..ab0dd3f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -133,6 +133,7 @@ dependencies = [ "similar", "strip-ansi-escapes", "tempfile", + "time", "tokio", "tracing", "tracing-subscriber", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 854fabcd..4e633236 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -53,3 +53,4 @@ serde_yaml = { workspace = true } similar = "2.7.0" strip-ansi-escapes = "0.2" tempfile = { workspace = true } +time = { version = "0.3.44", features = ["parsing"] } diff --git a/cli/src/mixed_run.rs b/cli/src/mixed_run.rs index 4bf93de0..cd655fcc 100644 --- a/cli/src/mixed_run.rs +++ b/cli/src/mixed_run.rs @@ -57,13 +57,13 @@ mod supervisor; pub(crate) use self::{launch_bundle::*, outside_proxy::*, supervisor::*}; const RECEIPT_SCHEMA: &str = "amber.run.receipt"; -const RECEIPT_VERSION: u32 = 3; +const RECEIPT_VERSION: u32 = 4; const LAUNCH_BUNDLE_SCHEMA: &str = "amber.run.launch_bundle"; const LAUNCH_BUNDLE_VERSION: u32 = 1; const SITE_STATE_SCHEMA: &str = "amber.run.site_state"; -const SITE_STATE_VERSION: u32 = 2; +const SITE_STATE_VERSION: u32 = 3; const SITE_PLAN_SCHEMA: &str = "amber.run.site_supervisor_plan"; -const SITE_PLAN_VERSION: u32 = 1; +const SITE_PLAN_VERSION: u32 = 2; const DESIRED_LINKS_SCHEMA: &str = "amber.run.desired_links"; const DESIRED_LINKS_VERSION: u32 = 1; const OTLP_SINK_PLAN_SCHEMA: &str = "amber.run.observability_sink"; @@ -99,7 +99,11 @@ pub(crate) fn amber_cli_executable() -> Result { let current = env::current_exe() .into_diagnostic() .wrap_err("failed to resolve amber executable path")?; - workspace_executable_from(¤t, "CARGO_BIN_EXE_amber", "amber", true) + amber_cli_executable_from(¤t) +} + +fn amber_cli_executable_from(current: &Path) -> Result { + workspace_executable_from(current, "CARGO_BIN_EXE_amber", "amber", true) } #[derive(Clone, Debug)] @@ -116,6 +120,18 @@ pub(crate) fn site_controller_command() -> Result { } fn site_controller_command_from(current: &Path) -> Result { + if let Some(executable) = cargo_binary_override("CARGO_BIN_EXE_amber-site-controller") { + return Ok(SiteControllerCommand { + executable, + prefix_args: Vec::new(), + }); + } + if launched_from_cargo_test_binary(current) { + return Ok(SiteControllerCommand { + executable: amber_cli_executable_from(current)?, + prefix_args: vec!["run-site-controller"], + }); + } match workspace_executable_from( current, "CARGO_BIN_EXE_amber-site-controller", @@ -139,11 +155,8 @@ fn workspace_executable_from( binary_name: &str, allow_current: bool, ) -> Result { - if let Some(path) = env::var_os(env_var) { - let path = PathBuf::from(path); - if path.is_file() { - return Ok(path); - } + if let Some(path) = cargo_binary_override(env_var) { + return Ok(path); } let executable_name = format!("{binary_name}{}", std::env::consts::EXE_SUFFIX); @@ -167,6 +180,20 @@ fn workspace_executable_from( )) } +fn cargo_binary_override(env_var: &str) -> Option { + env::var_os(env_var) + .map(PathBuf::from) + .filter(|path| path.is_file()) +} + +fn launched_from_cargo_test_binary(current: &Path) -> bool { + current + .parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + == Some("deps") +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct RunReceipt { pub(crate) schema: String, diff --git a/cli/src/mixed_run/launch_bundle.rs b/cli/src/mixed_run/launch_bundle.rs index 0e5b98eb..b86cc112 100644 --- a/cli/src/mixed_run/launch_bundle.rs +++ b/cli/src/mixed_run/launch_bundle.rs @@ -16,6 +16,21 @@ fn site_controller_image_reference() -> Result { Ok(site_controller_image_reference_from_overrides(&overrides)) } +pub(super) fn site_controller_local_router_control(kind: SiteKind, artifact_dir: &Path) -> String { + match kind { + SiteKind::Direct => format!( + "unix://{}", + direct_current_control_socket_path(artifact_dir).display() + ), + SiteKind::Vm => format!( + "unix://{}", + vm_current_control_socket_path(artifact_dir).display() + ), + SiteKind::Compose => "unix:///amber/control/router-control.sock".to_string(), + SiteKind::Kubernetes => "amber-router:24100".to_string(), + } +} + pub(crate) fn dry_run_run_plan( source_plan_path: Option<&Path>, run_plan: &RunPlan, @@ -224,6 +239,7 @@ pub(super) fn materialize_launch_bundle( let route_listen_addr = site_controller_route_listen_addr(site.site_plan.site.kind).to_string(); let mut peer_site_router_urls = BTreeMap::new(); + let mut peer_router_identities = BTreeMap::new(); let mut peer_router_mesh_addrs = BTreeMap::new(); let mut controller_route_ports = Vec::new(); let mut controller_routes = Vec::new(); @@ -238,6 +254,14 @@ pub(super) fn materialize_launch_bundle( peer_site_id.clone(), format!("http://{peer_router_host}:{route_port}"), ); + peer_router_identities.insert( + peer_site_id.clone(), + planned_router_identity( + run_id, + &run_plan.mesh_scope, + &peer_site_plan.router_identity_id, + ), + ); let peer_router_mesh_addr = planned_router_mesh_addrs .get(peer_site_id) .expect("planned router mesh addr should exist for every site"); @@ -273,6 +297,8 @@ pub(super) fn materialize_launch_bundle( )?; } site.base_supervisor_plan.controller_route_ports = controller_route_ports; + let local_router_control = + site_controller_local_router_control(site.site_plan.site.kind, &site.artifact_dir); amber_site_controller::write_site_controller_plan( &site.controller_plan_path, @@ -284,12 +310,9 @@ pub(super) fn materialize_launch_bundle( &site.controller_url, &site.site_plan.router_identity_id, &peer_site_router_urls, + &peer_router_identities, &peer_router_mesh_addrs, - match site.site_plan.site.kind { - SiteKind::Compose => Some("unix:///amber/control/router-control.sock"), - SiteKind::Kubernetes => Some("amber-router:24100"), - SiteKind::Direct | SiteKind::Vm => None, - }, + Some(local_router_control.as_str()), planned_router_mesh_addrs.get(site_id).map(String::as_str), &site.controller_state_path, bundle_root, diff --git a/cli/src/mixed_run/supervisor.rs b/cli/src/mixed_run/supervisor.rs index 437ea738..4c050f3f 100644 --- a/cli/src/mixed_run/supervisor.rs +++ b/cli/src/mixed_run/supervisor.rs @@ -784,7 +784,10 @@ pub(super) async fn try_discover_site( try_discover_kubernetes_site(plan, runtime, stop_requested, run_root).await } }?; - if discovery.is_none() && plan.kind == SiteKind::Compose { + if discovery.is_none() + && plan.kind == SiteKind::Compose + && !compose_site_controller_started(plan)? + { runtime.site_started = false; } Ok(discovery) @@ -826,6 +829,9 @@ pub(super) async fn try_discover_direct_site( Ok(None) | Err(_) => return Ok(None), }; let router_addr = SocketAddr::from(([127, 0, 0, 1], router_mesh_port)); + if !local_site_controller_ready(plan, VM_LOCAL_TARGET_READY_TIMEOUT)? { + return Ok(None); + } return Ok(Some(RouterDiscovery { control_endpoint, router_identity, @@ -869,6 +875,9 @@ pub(super) async fn try_discover_vm_site( if !vm_component_targets_ready(plan, &artifact_dir)? { return Ok(None); } + if !local_site_controller_ready(plan, VM_LOCAL_TARGET_READY_TIMEOUT)? { + return Ok(None); + } Ok(Some(RouterDiscovery { control_endpoint, router_identity, @@ -922,18 +931,127 @@ pub(super) fn mesh_config_local_targets_ready(path: &Path, timeout: Duration) -> Ok(true) } +pub(super) fn local_site_controller_ready( + plan: &SiteSupervisorPlan, + timeout: Duration, +) -> Result { + let Some(addr) = local_site_controller_addr(plan)? else { + return Ok(false); + }; + Ok(wait_for_http_response(addr, timeout).is_ok()) +} + +pub(super) fn local_site_controller_addr(plan: &SiteSupervisorPlan) -> Result> { + let Some(url) = plan.site_controller_url.as_deref() else { + return Ok(None); + }; + let url = Url::parse(url) + .into_diagnostic() + .wrap_err_with(|| format!("invalid site controller url `{url}`"))?; + if url.scheme() != "http" { + return Ok(None); + } + let host = url.host_str().ok_or_else(|| { + miette::miette!( + "site controller url `{url}` for site `{}` is missing a host", + plan.site_id + ) + })?; + let ip = match host { + "localhost" => std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), + _ => host.parse().into_diagnostic().wrap_err_with(|| { + format!( + "site controller url `{url}` for site `{}` has a non-IP host `{host}`", + plan.site_id + ) + })?, + }; + if !ip.is_loopback() { + return Ok(None); + } + let port = url.port_or_known_default().ok_or_else(|| { + miette::miette!( + "site controller url `{url}` for site `{}` is missing a port", + plan.site_id + ) + })?; + Ok(Some(SocketAddr::new(ip, port))) +} + +pub(super) fn compose_site_controller_container_name(plan: &SiteSupervisorPlan) -> Option { + (plan.kind == SiteKind::Compose) + .then_some(plan.compose_project.as_deref()?) + .map(|project| { + format!( + "{project}-{}-1", + amber_site_controller::SITE_CONTROLLER_SERVICE_NAME + ) + }) +} + +pub(super) fn parse_container_runtime_status(raw: &str) -> Option<(&str, Option<&str>)> { + let mut parts = raw.split_whitespace(); + let status = parts.next()?; + let health = parts.next(); + Some((status, health)) +} + +fn inspect_compose_site_controller_status( + plan: &SiteSupervisorPlan, +) -> Result)>> { + let Some(container_name) = compose_site_controller_container_name(plan) else { + return Ok(None); + }; + let output = Command::new("docker") + .arg("inspect") + .arg("--format") + .arg("{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}") + .arg(&container_name) + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to inspect compose site controller container `{container_name}`") + })?; + if !output.status.success() { + return Ok(None); + } + let stdout = String::from_utf8_lossy(&output.stdout); + Ok(parse_container_runtime_status(&stdout) + .map(|(status, health)| (status.to_string(), health.map(str::to_string)))) +} + +fn compose_site_controller_started(plan: &SiteSupervisorPlan) -> Result { + Ok(inspect_compose_site_controller_status(plan)? + .is_some_and(|(status, _)| matches!(status.as_str(), "created" | "running" | "restarting"))) +} + +fn compose_site_controller_ready(plan: &SiteSupervisorPlan) -> Result { + Ok( + inspect_compose_site_controller_status(plan)?.is_some_and(|(status, health)| { + status == "running" && health.as_deref().is_none_or(|health| health == "healthy") + }), + ) +} + pub(super) async fn try_discover_compose_site( plan: &SiteSupervisorPlan, stop_requested: &AtomicBool, run_root: &Path, ) -> Result> { - run_until_stop( + let Some(discovery) = run_until_stop( run_root, stop_requested, discover_router_for_output(&plan.artifact_dir, plan.compose_project.as_deref(), true), ) .await - .wrap_err_with(|| format!("compose router discovery for site `{}`", plan.site_id)) + .wrap_err_with(|| format!("compose router discovery for site `{}`", plan.site_id))? + else { + return Ok(None); + }; + if !compose_site_controller_ready(plan)? { + return Ok(None); + } + Ok(Some(discovery)) } pub(super) async fn try_discover_kubernetes_site( diff --git a/cli/src/mixed_run/tests.rs b/cli/src/mixed_run/tests.rs index 0d5700fe..cdc50696 100644 --- a/cli/src/mixed_run/tests.rs +++ b/cli/src/mixed_run/tests.rs @@ -1,3 +1,12 @@ +use std::{ + fs, + net::{SocketAddr, TcpListener}, + thread, + time::Duration, +}; + +use tempfile::TempDir; + use super::*; #[test] @@ -34,3 +43,187 @@ fn site_controller_image_override_uses_dev_tag() { ) ); } + +#[test] +fn site_controller_local_router_control_uses_backend_local_control_targets() { + let artifact_dir = Path::new("/tmp/site-artifact"); + + assert_eq!( + launch_bundle::site_controller_local_router_control(SiteKind::Compose, artifact_dir), + "unix:///amber/control/router-control.sock" + ); + assert_eq!( + launch_bundle::site_controller_local_router_control(SiteKind::Kubernetes, artifact_dir), + "amber-router:24100" + ); + assert!( + launch_bundle::site_controller_local_router_control(SiteKind::Direct, artifact_dir) + .starts_with("unix://"), + "direct site controllers should know their local router control socket up front" + ); + assert!( + launch_bundle::site_controller_local_router_control(SiteKind::Vm, artifact_dir) + .starts_with("unix://"), + "vm site controllers should know their local router control socket up front" + ); +} + +#[test] +fn site_controller_command_prefers_fresh_amber_binary_under_cargo_tests() { + let temp = TempDir::new().expect("temp dir"); + let debug_dir = temp.path().join("target").join("debug"); + let deps_dir = debug_dir.join("deps"); + fs::create_dir_all(&deps_dir).expect("deps dir should exist"); + let current = deps_dir.join(format!("mixed_run-test{}", std::env::consts::EXE_SUFFIX)); + let amber = debug_dir.join(format!("amber{}", std::env::consts::EXE_SUFFIX)); + let site_controller = debug_dir.join(format!( + "amber-site-controller{}", + std::env::consts::EXE_SUFFIX + )); + fs::write(¤t, "").expect("current test binary should exist"); + fs::write(&amber, "").expect("amber binary should exist"); + fs::write(&site_controller, "").expect("stale site controller binary should exist"); + + let command = site_controller_command_from(¤t).expect("site controller command"); + assert_eq!( + command.executable, amber, + "cargo test binaries should launch the freshly built amber binary instead of a stale \ + sibling amber-site-controller executable" + ); + assert_eq!(command.prefix_args, vec!["run-site-controller"]); +} + +#[test] +fn local_site_controller_addr_requires_loopback_http() { + let mut plan = SiteSupervisorPlan { + schema: "amber.run.site_supervisor_plan".to_string(), + version: 2, + run_id: "run".to_string(), + mesh_scope: "scope".to_string(), + run_root: "/tmp/run".to_string(), + coordinator_pid: 1, + site_id: "site-a".to_string(), + kind: SiteKind::Direct, + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), + storage_root: None, + runtime_root: None, + router_mesh_port: None, + compose_project: None, + kubernetes_namespace: None, + context: None, + port_forward_mesh_port: None, + port_forward_control_port: None, + observability_endpoint: None, + site_controller_plan_path: None, + site_controller_url: Some("http://127.0.0.1:24200".to_string()), + controller_route_ports: Vec::new(), + launch_env: BTreeMap::new(), + }; + + assert_eq!( + supervisor::local_site_controller_addr(&plan).expect("loopback controller address"), + Some(SocketAddr::from(([127, 0, 0, 1], 24200))) + ); + + plan.site_controller_url = Some("http://192.168.1.10:24200".to_string()); + assert_eq!( + supervisor::local_site_controller_addr(&plan) + .expect("non-loopback controller address should be ignored"), + None + ); +} + +#[test] +fn local_site_controller_ready_waits_for_http_listener() { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("listener should bind"); + let addr = listener.local_addr().expect("listener addr"); + let handle = thread::spawn(move || { + if let Ok((mut stream, _)) = listener.accept() { + let mut request = [0u8; 256]; + let _ = stream.read(&mut request); + let _ = stream.write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 11\r\nConnection: close\r\n\r\n{\"ok\":true}", + ); + } + }); + let plan = SiteSupervisorPlan { + schema: "amber.run.site_supervisor_plan".to_string(), + version: 2, + run_id: "run".to_string(), + mesh_scope: "scope".to_string(), + run_root: "/tmp/run".to_string(), + coordinator_pid: 1, + site_id: "site-a".to_string(), + kind: SiteKind::Direct, + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), + storage_root: None, + runtime_root: None, + router_mesh_port: None, + compose_project: None, + kubernetes_namespace: None, + context: None, + port_forward_mesh_port: None, + port_forward_control_port: None, + observability_endpoint: None, + site_controller_plan_path: None, + site_controller_url: Some(format!("http://127.0.0.1:{}", addr.port())), + controller_route_ports: Vec::new(), + launch_env: BTreeMap::new(), + }; + + assert!( + supervisor::local_site_controller_ready(&plan, Duration::from_secs(1)) + .expect("controller readiness should succeed"), + "local site readiness should wait for the controller HTTP listener" + ); + handle.join().expect("listener thread should exit"); +} + +#[test] +fn compose_site_controller_status_parsing_handles_health_and_plain_running() { + assert_eq!( + supervisor::parse_container_runtime_status("running healthy\n"), + Some(("running", Some("healthy"))) + ); + assert_eq!( + supervisor::parse_container_runtime_status("running\n"), + Some(("running", None)) + ); + assert_eq!(supervisor::parse_container_runtime_status(""), None); +} + +#[test] +fn compose_site_controller_container_name_uses_compose_project() { + let plan = SiteSupervisorPlan { + schema: "amber.run.site_supervisor_plan".to_string(), + version: 2, + run_id: "run".to_string(), + mesh_scope: "scope".to_string(), + run_root: "/tmp/run".to_string(), + coordinator_pid: 1, + site_id: "compose-site".to_string(), + kind: SiteKind::Compose, + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), + storage_root: None, + runtime_root: None, + router_mesh_port: None, + compose_project: Some("amber_run_compose-site".to_string()), + kubernetes_namespace: None, + context: None, + port_forward_mesh_port: None, + port_forward_control_port: None, + observability_endpoint: None, + site_controller_plan_path: None, + site_controller_url: Some("http://amber-site-controller:4100".to_string()), + controller_route_ports: vec![60066], + launch_env: BTreeMap::new(), + }; + + assert_eq!( + supervisor::compose_site_controller_container_name(&plan).as_deref(), + Some("amber_run_compose-site-amber-site-controller-1") + ); +} diff --git a/cli/tests/mixed_run/support.rs b/cli/tests/mixed_run/support.rs index a25336a9..5a014193 100644 --- a/cli/tests/mixed_run/support.rs +++ b/cli/tests/mixed_run/support.rs @@ -29,6 +29,7 @@ use cloud_image_support::default_host_arch_cloud_image_filename; use outputs_root_support::cli_test_outputs_root; use serde_json::{Value, json}; use target_dir_support::cargo_target_dir; +use time::{OffsetDateTime, format_description::well_known::Rfc3339}; pub(crate) use workspace_root_support::workspace_root; const COMMON_HTTP_APP: &str = r#"import json @@ -630,7 +631,7 @@ pub(crate) fn ensure_docker_image(tag: &str, dockerfile: &Path) { return; } - if image_platform_opt(tag).is_some() { + if docker_image_is_fresh(tag, dockerfile) { return; } @@ -664,6 +665,92 @@ pub(crate) fn ensure_docker_image(tag: &str, dockerfile: &Path) { panic!("docker build failed for {tag} after retries with status {status}"); } +fn docker_image_is_fresh(tag: &str, dockerfile: &Path) -> bool { + if image_platform_opt(tag).is_none() { + return false; + } + let Some(image_created) = docker_image_created_at(tag) else { + return false; + }; + !source_is_newer_than_image( + image_created, + newest_internal_image_input_mtime(&workspace_root(), dockerfile), + ) +} + +fn docker_image_created_at(tag: &str) -> Option { + let output = Command::new("docker") + .arg("image") + .arg("inspect") + .arg("-f") + .arg("{{.Created}}") + .arg(tag) + .output() + .ok()?; + if !output.status.success() { + return None; + } + let created = String::from_utf8_lossy(&output.stdout).trim().to_string(); + let timestamp = OffsetDateTime::parse(&created, &Rfc3339).ok()?; + Some(UNIX_EPOCH + Duration::from_secs(timestamp.unix_timestamp() as u64)) +} + +fn newest_internal_image_input_mtime( + workspace_root: &Path, + dockerfile: &Path, +) -> Option { + [ + workspace_root.join("Cargo.toml"), + workspace_root.join("Cargo.lock"), + workspace_root.join("rust-toolchain.toml"), + workspace_root.join("cli"), + workspace_root.join("compiler"), + workspace_root.join("images"), + workspace_root.join("runtime"), + dockerfile.to_path_buf(), + ] + .into_iter() + .filter_map(|path| newest_path_mtime(&path)) + .max() +} + +fn newest_path_mtime(path: &Path) -> Option { + let metadata = fs::metadata(path).ok()?; + if metadata.is_file() { + return metadata.modified().ok(); + } + if !metadata.is_dir() { + return None; + } + + let mut newest = metadata.modified().ok(); + let entries = fs::read_dir(path).ok()?; + for entry in entries.filter_map(Result::ok) { + let child = entry.path(); + let child_name = child + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(""); + if child_name == "target" || child_name == ".git" { + continue; + } + if let Some(child_mtime) = newest_path_mtime(&child) { + newest = match newest { + Some(current) if current >= child_mtime => Some(current), + _ => Some(child_mtime), + }; + } + } + newest +} + +fn source_is_newer_than_image( + image_created: SystemTime, + newest_source: Option, +) -> bool { + newest_source.is_some_and(|source| source > image_created) +} + pub(crate) fn docker_supports_buildx() -> bool { static READY: OnceLock = OnceLock::new(); *READY.get_or_init(|| { @@ -2158,3 +2245,25 @@ pub(crate) fn namespace_exists(namespace: &str, kubeconfig: &Path, context: &str .map(|output| output.status.success()) .unwrap_or(false) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn source_is_newer_than_image_detects_newer_sources() { + let image_created = UNIX_EPOCH + Duration::from_secs(10); + let newest_source = Some(UNIX_EPOCH + Duration::from_secs(11)); + assert!(source_is_newer_than_image(image_created, newest_source)); + } + + #[test] + fn source_is_newer_than_image_ignores_older_or_missing_sources() { + let image_created = UNIX_EPOCH + Duration::from_secs(10); + assert!(!source_is_newer_than_image( + image_created, + Some(UNIX_EPOCH + Duration::from_secs(9)) + )); + assert!(!source_is_newer_than_image(image_created, None)); + } +} diff --git a/compiler/src/targets/mesh/mesh_config.rs b/compiler/src/targets/mesh/mesh_config.rs index 3b8ffad7..37e49fb7 100644 --- a/compiler/src/targets/mesh/mesh_config.rs +++ b/compiler/src/targets/mesh/mesh_config.rs @@ -506,45 +506,6 @@ pub(crate) fn build_mesh_config_plan( }); } - let framework_bindings = mesh_plan - .framework_bindings() - .filter(|binding| binding.capability.as_str() == "component") - .collect::>(); - for binding in framework_bindings { - let slot_decl = scenario - .component(binding.consumer) - .slots - .get(binding.slot.as_str()) - .expect("framework binding target slot should exist"); - let consumer_id = identities_by_component - .get(&binding.consumer) - .expect("framework binding consumer identity missing") - .id - .clone(); - let authority_moniker = scenario.component(binding.authority_realm).moniker.as_str(); - let consumer_moniker = scenario.component(binding.consumer).moniker.as_str(); - let route_id = framework_cap_instance_id( - authority_moniker, - consumer_moniker, - &binding.consumer.0.to_string(), - &binding.slot, - binding.capability.as_str(), - ); - inbound.push(InboundRoute { - route_id, - capability: binding.capability.to_string(), - capability_kind: Some(slot_decl.decl.kind.to_string()), - capability_profile: slot_decl.decl.profile.clone(), - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV.to_string(), - optional: false, - }, - allowed_issuers: vec![consumer_id], - }); - } - let mesh_listen = format!("{}:{router_mesh_port}", options.router_mesh_listen_addr) .parse() .expect("mesh listen"); @@ -652,7 +613,7 @@ mod tests { sync::Arc, }; - use amber_manifest::{ManifestDigest, ProvideDecl, SlotDecl}; + use amber_manifest::{FrameworkCapabilityName, ManifestDigest, ProvideDecl, SlotDecl}; use amber_scenario::{Component, ComponentId, Moniker, ProvideRef, Scenario, ScenarioExport}; use serde_json::json; @@ -661,6 +622,7 @@ mod tests { mesh::{ plan::{ EndpointInfo, MeshPlan, ResolvedBinding, ResolvedComponentBinding, ResolvedExport, + ResolvedFrameworkBinding, }, ports::allocate_local_route_ports, }, @@ -848,4 +810,94 @@ mod tests { "unused local routes must stay closed to the router and should not be emitted", ); } + + #[test] + fn build_mesh_config_plan_leaves_framework_component_routes_to_site_controller_overlays() { + let mut consumer = component( + 0, + "/consumer", + json!({ + "image": "consumer", + "entrypoint": ["consumer"], + }), + ); + consumer.slots.insert( + "realm".to_string(), + serde_json::from_value::(json!({ + "kind": "component", + })) + .expect("slot decl"), + ); + + let scenario = Scenario { + root: ComponentId(0), + components: vec![Some(consumer)], + bindings: Vec::new(), + exports: Vec::new(), + manifest_catalog: BTreeMap::new(), + }; + + let mesh_plan = MeshPlan::new( + vec![ComponentId(0)], + vec![ResolvedBinding::Framework(ResolvedFrameworkBinding { + consumer: ComponentId(0), + slot: "realm".to_string(), + authority_realm: ComponentId(0), + capability: FrameworkCapabilityName::try_from("component") + .expect("framework capability"), + })], + Vec::new(), + HashMap::new(), + ); + + let endpoint_plan = build_endpoint_plan(&scenario).expect("endpoint plan"); + let route_ports = allocate_local_route_ports(&scenario, &endpoint_plan, &mesh_plan) + .expect("local route ports"); + let mesh_ports_by_component = HashMap::from([(ComponentId(0), 23000)]); + + let plan = build_mesh_config_plan(MeshConfigBuildInput { + scenario: &scenario, + mesh_plan: &mesh_plan, + route_ports: &route_ports, + mesh_ports_by_component: &mesh_ports_by_component, + router_ports: Some(RouterPorts { + mesh: 24000, + control: 24100, + }), + addressing: &StaticAddressing, + options: MeshConfigBuildOptions { + router_identity_id: "/site/test/router", + ..default_mesh_config_build_options() + }, + }) + .expect("mesh config plan"); + + let component_config = plan + .component_configs + .get(&ComponentId(0)) + .expect("component config"); + assert!( + component_config.outbound.iter().any(|route| { + route.slot == "realm" + && route.capability == "component" + && route.peer_id == "/site/test/router" + && route.peer_addr == "router:24000" + }), + "framework.component consumers should still send requests to the local router", + ); + + let router_config = plan.router_config.expect("router config"); + assert!( + router_config.inbound.iter().all(|route| { + route.capability != "component" + || !matches!( + route.target, + InboundTarget::External { ref url_env, optional } + if url_env == FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV && !optional + ) + }), + "static router mesh config must not bake in framework.component delivery; the site \ + controller overlay owns those routes", + ); + } } diff --git a/docker/images.json b/docker/images.json index b982d2e8..43dca0d5 100644 --- a/docker/images.json +++ b/docker/images.json @@ -17,7 +17,7 @@ "name": "amber-router", "context": ".", "dockerfile": "docker/amber-router/Dockerfile", - "version": "v0.1.x" + "version": "v0.2.x" }, { "name": "amber-docker-gateway", diff --git a/manager/src/runtime.rs b/manager/src/runtime.rs index e5e6cc8a..d54399fb 100644 --- a/manager/src/runtime.rs +++ b/manager/src/runtime.rs @@ -391,7 +391,9 @@ fn is_expected_exited_service(service: &ComposePsEntry) -> bool { return false; } - EXPECTED_EXITED_SERVICES.contains(&name) || name.ends_with("-egress-init") + EXPECTED_EXITED_SERVICES.contains(&name) + || name.ends_with("-egress-init") + || name.ends_with("-control-init") } fn is_expected_exited_state(state: &str) -> bool { @@ -611,6 +613,17 @@ mod tests { ); } + #[test] + fn compose_health_treats_successful_control_init_as_healthy() { + assert_eq!( + classify_compose_services(&[ + service("controller", "running"), + service("c0-component-net-control-init", "exited (0)"), + ]), + ScenarioHealth::Healthy + ); + } + #[test] fn compose_health_treats_failed_egress_init_as_failed() { assert_eq!( @@ -621,6 +634,16 @@ mod tests { ); } + #[test] + fn compose_health_treats_failed_control_init_as_failed() { + assert_eq!( + classify_compose_services(&[service("c0-component-net-control-init", "exited (1)")]), + ScenarioHealth::Failed( + "service c0-component-net-control-init is exited (1)".to_string() + ) + ); + } + #[test] fn compose_health_treats_exited_primary_service_as_failed() { assert_eq!( diff --git a/runtime/router/src/control.rs b/runtime/router/src/control.rs index 6b3d2478..90d12d34 100644 --- a/runtime/router/src/control.rs +++ b/runtime/router/src/control.rs @@ -558,6 +558,7 @@ pub(super) async fn handle_inbound( &mut session, outbound, route.route_id.clone().into(), + remote_id.clone().into(), plugins, labels, ) @@ -677,6 +678,7 @@ pub(super) async fn handle_outbound( proxy_local_http_to_noise( &mut outbound, route.route_id.clone().into(), + config.identity.id.clone().into(), stream, plugins, HttpExchangeLabels::outbound_from_route(config.identity.id.clone().into(), &route), diff --git a/runtime/router/src/dynamic_caps.rs b/runtime/router/src/dynamic_caps.rs index e556d04f..1b6dcf08 100644 --- a/runtime/router/src/dynamic_caps.rs +++ b/runtime/router/src/dynamic_caps.rs @@ -1079,6 +1079,7 @@ impl DynamicCapsRuntime { Arc::from(Vec::>::new()) }, route_id: upstream.route_id.clone(), + peer_id: Arc::::from(self.config.identity.id.as_str()), labels: HttpExchangeLabels { kind: HttpEdgeKind::Binding, emit_telemetry: true, diff --git a/runtime/router/src/lib.rs b/runtime/router/src/lib.rs index eb12feca..5f5063dd 100644 --- a/runtime/router/src/lib.rs +++ b/runtime/router/src/lib.rs @@ -319,6 +319,7 @@ struct OutboundHttpProxyState { upstream: Arc>>, plugins: Arc<[Arc]>, route_id: Arc, + peer_id: Arc, labels: HttpExchangeLabels, dynamic_caps: Option>, } @@ -1233,6 +1234,7 @@ async fn proxy_local_http_request( async fn proxy_local_http_to_noise( session: &mut NoiseSession, route_id: Arc, + peer_id: Arc, stream: tokio::net::TcpStream, plugins: Arc<[Arc]>, labels: HttpExchangeLabels, @@ -1259,6 +1261,7 @@ async fn proxy_local_http_to_noise( upstream: Arc::new(Mutex::new(sender)), plugins, route_id, + peer_id, labels, dynamic_caps, }; @@ -1470,6 +1473,16 @@ async fn proxy_http_request_to_noise( } let host_header = outgoing_host_header(&request_parts.uri, &request_parts.headers); sanitize_request_headers(&mut request_parts.headers, host_header.as_str()); + request_parts.headers.insert( + header::HeaderName::from_static(AMBER_ROUTE_ID_HEADER), + HeaderValue::from_str(state.route_id.as_ref()) + .expect("route id header value should be valid"), + ); + request_parts.headers.insert( + header::HeaderName::from_static(AMBER_PEER_ID_HEADER), + HeaderValue::from_str(state.peer_id.as_ref()) + .expect("peer id header value should be valid"), + ); inject_trace_context(&span, &mut request_parts.headers); let upstream_uri = request_parts.uri.to_string(); @@ -1726,14 +1739,24 @@ async fn proxy_http_request(state: HttpProxyState, req: Request) -> Re }; sanitize_request_headers(&mut parts.0.headers, &host_header); - if let Some(route_id) = state.route_id.as_ref() { + if let Some(route_id) = state.route_id.as_ref() + && !parts + .0 + .headers + .contains_key(header::HeaderName::from_static(AMBER_ROUTE_ID_HEADER)) + { parts.0.headers.insert( header::HeaderName::from_static(AMBER_ROUTE_ID_HEADER), HeaderValue::from_str(route_id.as_ref()) .expect("route id header value should be valid"), ); } - if let Some(peer_id) = state.peer_id.as_ref() { + if let Some(peer_id) = state.peer_id.as_ref() + && !parts + .0 + .headers + .contains_key(header::HeaderName::from_static(AMBER_PEER_ID_HEADER)) + { parts.0.headers.insert( header::HeaderName::from_static(AMBER_PEER_ID_HEADER), HeaderValue::from_str(peer_id.as_ref()) diff --git a/runtime/router/src/tests.rs b/runtime/router/src/tests.rs index 51997ab7..cbdaa459 100644 --- a/runtime/router/src/tests.rs +++ b/runtime/router/src/tests.rs @@ -2173,3 +2173,223 @@ async fn late_mesh_slot_registration_succeeds_on_same_http_connection() { let _ = proxy_task.await; let _ = mesh_server_task.await; } + +#[tokio::test] +async fn outbound_http_proxy_overwrites_internal_framework_auth_headers() { + let captured = Arc::new(StdMutex::new(None::<(String, String)>)); + let (upstream_client, upstream_server) = duplex(64 * 1024); + let captured_upstream = captured.clone(); + let upstream_task = tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let captured = captured_upstream.clone(); + async move { + let route_id = req + .headers() + .get(AMBER_ROUTE_ID_HEADER) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string(); + let peer_id = req + .headers() + .get(AMBER_PEER_ID_HEADER) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string(); + *captured.lock().expect("capture lock") = Some((route_id, peer_id)); + Ok::<_, std::convert::Infallible>( + Response::builder() + .status(StatusCode::OK) + .body(empty_box_body()) + .expect("upstream response should build"), + ) + } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(upstream_server), service) + .await + .expect("upstream server should complete"); + }); + + let (sender, conn) = client_http1::handshake(TokioIo::new(upstream_client)) + .await + .expect("upstream handshake should succeed"); + let upstream_conn_task = tokio::spawn(async move { + conn.await.expect("upstream connection should complete"); + }); + + let state = OutboundHttpProxyState { + upstream: Arc::new(Mutex::new(sender)), + plugins: Arc::from(Vec::>::new()), + route_id: Arc::::from("framework-route"), + peer_id: Arc::::from("/component/delegate"), + labels: test_http_exchange_labels(), + dynamic_caps: None, + }; + + let (proxy_client, proxy_server) = duplex(64 * 1024); + let proxy_task = tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let state = state.clone(); + async move { + Ok::<_, std::convert::Infallible>(proxy_outbound_http_request(state, req).await) + } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(proxy_server), service) + .await + .expect("proxy server should complete"); + }); + + let (mut client, conn) = client_http1::handshake(TokioIo::new(proxy_client)) + .await + .expect("proxy handshake should succeed"); + let proxy_conn_task = tokio::spawn(async move { + conn.await.expect("proxy connection should complete"); + }); + + let response = client + .send_request( + Request::builder() + .uri("/framework/create") + .header(AMBER_ROUTE_ID_HEADER, "spoofed-route") + .header(AMBER_PEER_ID_HEADER, "/spoofed/peer") + .body(empty_box_body()) + .expect("request should build"), + ) + .await + .expect("proxy request should succeed"); + assert_eq!(response.status(), StatusCode::OK); + drop(response); + drop(client); + + assert_eq!( + *captured.lock().expect("capture lock"), + Some(( + "framework-route".to_string(), + "/component/delegate".to_string(), + )), + "router-generated internal auth headers must overwrite caller-supplied values", + ); + + proxy_task.await.expect("proxy task should join"); + proxy_conn_task + .await + .expect("proxy connection task should join"); + upstream_task.await.expect("upstream task should join"); + upstream_conn_task + .await + .expect("upstream connection task should join"); +} + +#[tokio::test] +async fn framework_external_proxy_preserves_forwarded_internal_auth_headers() { + let captured = Arc::new(StdMutex::new(None::<(String, String)>)); + let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) + .await + .expect("controller listener should bind"); + let addr = listener.local_addr().expect("listener addr"); + let captured_upstream = captured.clone(); + let upstream_task = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("controller should accept"); + let service = service_fn(move |req: Request| { + let captured = captured_upstream.clone(); + async move { + let route_id = req + .headers() + .get(AMBER_ROUTE_ID_HEADER) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string(); + let peer_id = req + .headers() + .get(AMBER_PEER_ID_HEADER) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string(); + *captured.lock().expect("capture lock") = Some((route_id, peer_id)); + Ok::<_, std::convert::Infallible>( + Response::builder() + .status(StatusCode::OK) + .body(empty_box_body()) + .expect("controller response should build"), + ) + } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await + .expect("controller server should complete"); + }); + + let (client, vetted_external_addrs) = build_client(); + let external_overrides: ExternalOverrides = Arc::new(RwLock::new(HashMap::from([( + "framework".to_string(), + format!("http://127.0.0.1:{}/", addr.port()), + )]))); + let state = HttpProxyState { + client, + target: ExternalTarget { + name: "framework".to_string(), + url_env: amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV.to_string(), + optional: false, + url_override: None, + }, + labels: test_http_exchange_labels(), + config: Arc::new(test_mesh_config()), + external_overrides, + vetted_external_addrs, + mesh_upstream: Arc::new(Mutex::new(None)), + route_id: Some(Arc::::from("site-router-route")), + peer_id: Some(Arc::::from("/site/source/router")), + }; + + let (proxy_client, proxy_server) = duplex(64 * 1024); + let proxy_task = tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let state = state.clone(); + async move { Ok::<_, std::convert::Infallible>(proxy_http_request(state, req).await) } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(proxy_server), service) + .await + .expect("proxy server should complete"); + }); + + let (mut client, conn) = client_http1::handshake(TokioIo::new(proxy_client)) + .await + .expect("proxy handshake should succeed"); + let proxy_conn_task = tokio::spawn(async move { + conn.await.expect("proxy connection should complete"); + }); + + let response = client + .send_request( + Request::builder() + .uri("/v1/children") + .header(AMBER_ROUTE_ID_HEADER, "forwarded-framework-route") + .header(AMBER_PEER_ID_HEADER, "/component/delegate") + .body(empty_box_body()) + .expect("request should build"), + ) + .await + .expect("proxy request should succeed"); + assert_eq!(response.status(), StatusCode::OK); + drop(response); + drop(client); + + assert_eq!( + *captured.lock().expect("capture lock"), + Some(( + "forwarded-framework-route".to_string(), + "/component/delegate".to_string(), + )), + "destination router must preserve forwarded framework auth headers instead of replacing \ + them with the intermediate site-router identity", + ); + + proxy_task.await.expect("proxy task should join"); + proxy_conn_task + .await + .expect("proxy connection task should join"); + upstream_task.await.expect("upstream task should join"); +} diff --git a/runtime/router/src/transport.rs b/runtime/router/src/transport.rs index 179b48b5..69fe05c4 100644 --- a/runtime/router/src/transport.rs +++ b/runtime/router/src/transport.rs @@ -421,6 +421,7 @@ pub(super) async fn proxy_noise_to_noise_http( session: &mut NoiseSession, outbound: NoiseSession, route_id: Arc, + peer_id: Arc, plugins: Arc<[Arc]>, labels: HttpExchangeLabels, ) -> Result<(), RouterError> { @@ -453,6 +454,7 @@ pub(super) async fn proxy_noise_to_noise_http( upstream: Arc::new(Mutex::new(sender)), plugins, route_id, + peer_id, labels, dynamic_caps: None, }; diff --git a/runtime/site-controller/src/default_runtime.rs b/runtime/site-controller/src/default_runtime.rs index 6bae58ed..2bd5519f 100644 --- a/runtime/site-controller/src/default_runtime.rs +++ b/runtime/site-controller/src/default_runtime.rs @@ -92,7 +92,7 @@ const DYNAMIC_COMPOSE_CHILD_VERSION: u32 = 1; const DYNAMIC_COMPOSE_MESH_ROOT: &str = ".amber/mesh"; const DYNAMIC_ROUTE_OVERLAY_FILENAME: &str = "site-router-overlay.json"; const DYNAMIC_PROXY_EXPORTS_FILENAME: &str = "proxy-exports.json"; -const COMPONENT_CONTROL_SOCKET_PATH_IN_CONTAINER: &str = "/amber/control/router-control.sock"; +const COMPONENT_CONTROL_SOCKET_PATH_IN_VOLUME: &str = "/router-control.sock"; const COMPOSE_PROVISIONER_SERVICE_NAME: &str = "amber-provisioner"; const COMPOSE_ROUTER_SERVICE_NAME: &str = "amber-router"; const KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH: &str = "01-configmaps/amber-mesh-provision.yaml"; @@ -103,7 +103,7 @@ const KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH: &str = "02-rbac/amber-provisi const KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME: &str = "amber-router-external"; const KUBERNETES_ROUTER_COMPONENT_NAME: &str = "amber-router"; const SITE_PLAN_SCHEMA: &str = "amber.run.site_supervisor_plan"; -const SITE_PLAN_VERSION: u32 = 1; +const SITE_PLAN_VERSION: u32 = 2; const SITE_CONTROLLER_RUNTIME_STATE_SCHEMA: &str = "amber.run.site_controller_runtime_state"; const SITE_CONTROLLER_RUNTIME_STATE_VERSION: u32 = 1; const DESIRED_LINKS_SCHEMA: &str = "amber.run.desired_links"; diff --git a/runtime/site-controller/src/default_runtime/child_runtime.rs b/runtime/site-controller/src/default_runtime/child_runtime.rs index c8966f8e..1caa07df 100644 --- a/runtime/site-controller/src/default_runtime/child_runtime.rs +++ b/runtime/site-controller/src/default_runtime/child_runtime.rs @@ -59,6 +59,7 @@ pub(crate) struct BridgeProxyKey { #[derive(Clone, Default)] pub struct DefaultSiteControllerRuntime { bridge_proxies: Arc>>, + runtime_apps: Arc>>, } pub(crate) fn default_site_controller_runtime() -> SharedSiteControllerRuntime { @@ -72,6 +73,10 @@ impl SiteControllerRuntime for DefaultSiteControllerRuntime { let mut guard = self.bridge_proxies.lock().await; std::mem::take(&mut *guard) }; + self.runtime_apps + .lock() + .expect("site controller runtime app cache poisoned") + .clear(); stop_bridge_proxies(&mut bridge_proxies).await }) } @@ -104,7 +109,10 @@ impl SiteControllerRuntime for DefaultSiteControllerRuntime { state: FrameworkControlState, child: LiveChildRecord, ) -> SiteControllerRuntimeFuture<'a, ()> { - Box::pin(async move { site_controller_prepare_child(plan, state, child).await }) + Box::pin(async move { + let app = self.runtime_app(plan)?; + site_controller_runtime_prepare_child(&app, &state, &child).await + }) } fn publish_child<'a>( @@ -113,7 +121,10 @@ impl SiteControllerRuntime for DefaultSiteControllerRuntime { state: FrameworkControlState, child: LiveChildRecord, ) -> SiteControllerRuntimeFuture<'a, ()> { - Box::pin(async move { site_controller_publish_child(plan, state, child).await }) + Box::pin(async move { + let app = self.runtime_app(plan)?; + site_controller_runtime_publish_child(&app, &state, &child).await + }) } fn rollback_child<'a>( @@ -121,7 +132,10 @@ impl SiteControllerRuntime for DefaultSiteControllerRuntime { plan: &'a SiteControllerPlan, child_id: u64, ) -> SiteControllerRuntimeFuture<'a, ()> { - Box::pin(async move { site_controller_rollback_child(plan, child_id).await }) + Box::pin(async move { + let app = self.runtime_app(plan)?; + site_controller_runtime_rollback_child(&app, child_id).await + }) } fn destroy_child<'a>( @@ -130,13 +144,17 @@ impl SiteControllerRuntime for DefaultSiteControllerRuntime { state: FrameworkControlState, child: LiveChildRecord, ) -> SiteControllerRuntimeFuture<'a, ()> { - Box::pin(async move { site_controller_destroy_child(plan, state, child).await }) + Box::pin(async move { + let app = self.runtime_app(plan)?; + site_controller_runtime_destroy_child(&app, &state, &child).await + }) } fn collect_live_component_runtime_metadata( &self, plan: &SiteControllerRuntimePlan, ) -> Result> { + let _ = self.runtime_app_for_plan(plan)?; collect_live_component_runtime_metadata(plan) } @@ -191,6 +209,31 @@ impl SiteControllerRuntime for DefaultSiteControllerRuntime { } } +impl DefaultSiteControllerRuntime { + fn runtime_app(&self, plan: &SiteControllerPlan) -> Result { + self.runtime_app_for_plan( + &crate::runtime_api::site_controller_runtime_plan_from_controller_plan(plan), + ) + } + + fn runtime_app_for_plan( + &self, + plan: &SiteControllerRuntimePlan, + ) -> Result { + let key = Path::new(&plan.site_state_root).to_path_buf(); + let mut runtime_apps = self + .runtime_apps + .lock() + .expect("site controller runtime app cache poisoned"); + if let Some(app) = runtime_apps.get(&key) { + return Ok(app.clone()); + } + let app = build_site_controller_runtime_app(plan.clone())?; + runtime_apps.insert(key, app.clone()); + Ok(app) + } +} + pub(crate) async fn stop_bridge_proxies( bridge_proxies: &mut BTreeMap, ) -> Result<()> { @@ -226,49 +269,6 @@ pub(super) fn build_site_controller_runtime_app( }) } -pub(crate) async fn site_controller_prepare_child( - plan: &SiteControllerPlan, - state: FrameworkControlState, - child: LiveChildRecord, -) -> Result<()> { - let app = build_site_controller_runtime_app( - crate::runtime_api::site_controller_runtime_plan_from_controller_plan(plan), - )?; - site_controller_runtime_prepare_child(&app, &state, &child).await -} - -pub(crate) async fn site_controller_publish_child( - plan: &SiteControllerPlan, - state: FrameworkControlState, - child: LiveChildRecord, -) -> Result<()> { - let app = build_site_controller_runtime_app( - crate::runtime_api::site_controller_runtime_plan_from_controller_plan(plan), - )?; - site_controller_runtime_publish_child(&app, &state, &child).await -} - -pub(crate) async fn site_controller_rollback_child( - plan: &SiteControllerPlan, - child_id: u64, -) -> Result<()> { - let app = build_site_controller_runtime_app( - crate::runtime_api::site_controller_runtime_plan_from_controller_plan(plan), - )?; - site_controller_runtime_rollback_child(&app, child_id).await -} - -pub(crate) async fn site_controller_destroy_child( - plan: &SiteControllerPlan, - state: FrameworkControlState, - child: LiveChildRecord, -) -> Result<()> { - let app = build_site_controller_runtime_app( - crate::runtime_api::site_controller_runtime_plan_from_controller_plan(plan), - )?; - site_controller_runtime_destroy_child(&app, &state, &child).await -} - #[derive(Clone)] pub(super) struct SiteControllerRuntimeApp { pub(super) plan: SiteControllerRuntimePlan, diff --git a/runtime/site-controller/src/default_runtime/compose_controller.rs b/runtime/site-controller/src/default_runtime/compose_controller.rs index aa204f7f..5b02d9af 100644 --- a/runtime/site-controller/src/default_runtime/compose_controller.rs +++ b/runtime/site-controller/src/default_runtime/compose_controller.rs @@ -54,6 +54,17 @@ pub fn inject_compose_site_controller( "user": "0:0", "command": ["--plan", plan_path.display().to_string()], "networks": networks, + "extra_hosts": ["host.docker.internal:host-gateway"], + "healthcheck": { + "test": [ + "CMD-SHELL", + "wget -qO- http://127.0.0.1:4100/healthz | grep -q '\"ok\":true'" + ], + "interval": "2s", + "timeout": "2s", + "retries": 30, + "start_period": "1s" + }, "volumes": [ format!("{}:{}", plan.run_root, plan.run_root), format!( diff --git a/runtime/site-controller/src/default_runtime/dynamic_routes.rs b/runtime/site-controller/src/default_runtime/dynamic_routes.rs index b767c1de..efc7ca7f 100644 --- a/runtime/site-controller/src/default_runtime/dynamic_routes.rs +++ b/runtime/site-controller/src/default_runtime/dynamic_routes.rs @@ -1196,7 +1196,10 @@ pub(super) fn apply_dynamic_route_issuer_grants( } pub(super) fn dynamic_direct_input_overlay_id(component: &str) -> String { - format!("framework-direct-inputs:{component}") + format!( + "framework-direct-inputs:{}", + base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(component.as_bytes()) + ) } pub(super) fn dynamic_direct_input_grants( @@ -1524,6 +1527,20 @@ mod direct_input_tests { ); } + #[test] + fn dynamic_direct_input_overlay_id_is_path_safe_for_component_monikers() { + let overlay_id = dynamic_direct_input_overlay_id("/source"); + assert_eq!( + overlay_id, "framework-direct-inputs:L3NvdXJjZQ", + "overlay ids should encode component monikers so router control paths stay \ + single-segment", + ); + assert!( + !overlay_id.contains('/'), + "overlay ids must not contain path separators: {overlay_id}", + ); + } + #[test] fn cleanup_dynamic_site_children_removes_child_roots_and_clears_state() { let temp = tempfile::tempdir().expect("tempdir should be created"); @@ -1667,7 +1684,7 @@ mod direct_input_tests { r#" services: amber-router: - image: ghcr.io/rdi-foundation/amber-router:v0.1.x + image: ghcr.io/rdi-foundation/amber-router:v0.2.x ports: - "127.0.0.1::24000" "#, diff --git a/runtime/site-controller/src/default_runtime/site_artifacts.rs b/runtime/site-controller/src/default_runtime/site_artifacts.rs index a9017d09..6936b6b4 100644 --- a/runtime/site-controller/src/default_runtime/site_artifacts.rs +++ b/runtime/site-controller/src/default_runtime/site_artifacts.rs @@ -410,16 +410,19 @@ pub(super) fn local_kubernetes_peer_identities( pub(crate) fn collect_live_component_runtime_metadata( plan: &SiteControllerRuntimePlan, ) -> Result> { - let state: SiteControllerRuntimeState = read_json( - &site_controller_runtime_state_path(Path::new(&plan.site_state_root)), - "site controller runtime state", - )?; - let published_children = state - .children - .values() - .filter(|child| child.published) - .cloned() - .collect::>(); + let state_path = site_controller_runtime_state_path(Path::new(&plan.site_state_root)); + let published_children = if state_path.is_file() { + let state: SiteControllerRuntimeState = + read_json(&state_path, "site controller runtime state")?; + state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>() + } else { + Vec::new() + }; let mut components = match plan.kind { SiteKind::Direct => collect_direct_artifact_runtime_metadata( Path::new(&plan.artifact_dir), @@ -731,7 +734,7 @@ fn collect_compose_artifact_runtime_metadata( compose_project, service_name, ), - socket_path: COMPONENT_CONTROL_SOCKET_PATH_IN_CONTAINER.to_string(), + socket_path: COMPONENT_CONTROL_SOCKET_PATH_IN_VOLUME.to_string(), } }), mesh_config, @@ -743,6 +746,8 @@ fn collect_compose_artifact_runtime_metadata( #[cfg(test)] mod tests { + use amber_mesh::MeshConfigTemplate; + use super::*; #[test] @@ -784,6 +789,323 @@ mod tests { "vm runtime metadata should use the hashed short socket path: {rendered}", ); } + + #[test] + fn collect_direct_runtime_metadata_uses_hashed_sidecar_control_socket_path() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let artifact_root = temp.path().join("artifact"); + let runtime_root = temp.path().join("runtime"); + fs::create_dir_all(&artifact_root).expect("artifact root should create"); + fs::create_dir_all(runtime_root.join("work/components/c1-provider")) + .expect("runtime work dir should create"); + + write_json( + &artifact_root.join("direct-plan.json"), + &serde_json::json!({ + "version": "3", + "mesh_provision_plan": "mesh-provision-plan.json", + "startup_order": [1], + "components": [ + { + "id": 1, + "moniker": "/provider", + "log_name": "provider", + "sidecar": { + "log_name": "provider-sidecar", + "mesh_port": 24001, + "mesh_config_path": "provider-mesh.json", + "mesh_identity_path": "provider-identity.json", + }, + "program": { + "log_name": "provider-program", + "work_dir": "work/components/c1-provider", + "execution": { + "kind": "direct", + "entrypoint": ["/bin/true"], + }, + }, + } + ], + "router": { + "identity_id": "/site/direct/router", + "mesh_port": 24000, + "control_port": 24100, + "control_socket_path": "router.sock", + "mesh_config_path": "router-mesh.json", + "mesh_identity_path": "router-identity.json" + } + }), + ) + .expect("direct plan should write"); + write_json( + &direct_runtime_state_path(&artifact_root), + &DirectRuntimeState { + component_mesh_port_by_id: BTreeMap::from([(1, 24001)]), + ..Default::default() + }, + ) + .expect("direct runtime state should write"); + write_json( + &runtime_root.join("provider-mesh.json"), + &MeshConfigPublic { + identity: MeshIdentityPublic { + id: "/provider".to_string(), + public_key: [3; 32], + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:24001".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + ) + .expect("provider mesh config should write"); + + let metadata = collect_direct_artifact_runtime_metadata(&artifact_root, &runtime_root) + .expect("direct metadata should load"); + let provider = metadata.get("/provider").expect("provider metadata"); + let Some(ControlEndpoint::Unix(path)) = provider.control_endpoint.as_ref() else { + panic!("direct metadata should expose a unix control socket"); + }; + assert_eq!( + path, + &amber_mesh::stable_temp_socket_path( + "amber-direct-control", + "sidecar-1", + &runtime_root.join("work/components/c1-provider"), + ), + "direct runtime metadata should use the hashed short sidecar control socket path", + ); + } + + #[test] + fn collect_vm_runtime_metadata_uses_hashed_sidecar_control_socket_path() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let artifact_root = temp.path().join("artifact"); + let runtime_root = temp.path().join("runtime"); + fs::create_dir_all(&artifact_root).expect("artifact root should create"); + fs::create_dir_all(runtime_root.join("work/sidecars/vm-provider")) + .expect("runtime sidecar dir should create"); + + write_json( + &artifact_root.join("vm-plan.json"), + &serde_json::json!({ + "version": "1", + "mesh_provision_plan": "mesh-provision-plan.json", + "startup_order": [1], + "runtime_addresses": {}, + "components": [ + { + "id": 1, + "moniker": "/provider", + "log_name": "vm-provider", + "mesh_config_path": "provider-mesh.json", + "mesh_identity_path": "provider-identity.json", + "cpus": { "kind": "literal", "value": 1 }, + "memory_mib": { "kind": "literal", "value": 512 }, + "base_image": { "kind": "static", "path": "/tmp/base.img" }, + "egress": "none", + "storage_mounts": [], + } + ] + }), + ) + .expect("vm plan should write"); + write_vm_runtime_state( + &artifact_root, + &VmRuntimeState { + slot_ports_by_component: BTreeMap::new(), + slot_route_ports_by_component: BTreeMap::new(), + route_host_ports_by_component: BTreeMap::new(), + endpoint_forwards_by_component: BTreeMap::new(), + component_mesh_port_by_id: BTreeMap::from([(1, 24001)]), + router_mesh_port: None, + }, + ) + .expect("vm runtime state should write"); + write_json( + &runtime_root.join("provider-mesh.json"), + &MeshConfigPublic { + identity: MeshIdentityPublic { + id: "/provider".to_string(), + public_key: [5; 32], + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:24001".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + ) + .expect("provider mesh config should write"); + + let metadata = collect_vm_artifact_runtime_metadata(&artifact_root, &runtime_root) + .expect("vm metadata should load"); + let provider = metadata.get("/provider").expect("provider metadata"); + let Some(ControlEndpoint::Unix(path)) = provider.control_endpoint.as_ref() else { + panic!("vm metadata should expose a unix control socket"); + }; + assert_eq!( + path, + &amber_mesh::stable_temp_socket_path( + "amber-vm-control", + "sidecar-1", + &runtime_root.join("work/sidecars/vm-provider"), + ), + "vm runtime metadata should use the hashed short sidecar control socket path", + ); + } + + #[test] + fn collect_live_component_runtime_metadata_tolerates_missing_runtime_state_file() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let artifact_root = temp.path().join("artifact"); + let runtime_root = temp.path().join("runtime"); + let site_state_root = temp.path().join("state"); + fs::create_dir_all(&artifact_root).expect("artifact root should create"); + fs::create_dir_all(&runtime_root).expect("runtime root should create"); + fs::create_dir_all(&site_state_root).expect("site state root should create"); + + write_json( + &artifact_root.join("direct-plan.json"), + &serde_json::json!({ + "version": "3", + "mesh_provision_plan": "mesh-provision-plan.json", + "startup_order": [], + "components": [], + "router": { + "identity_id": "/site/direct/router", + "mesh_port": 24000, + "control_port": 24100, + "control_socket_path": "router.sock", + "mesh_config_path": "router-mesh.json", + "mesh_identity_path": "router-identity.json" + } + }), + ) + .expect("direct plan should write"); + write_json( + &direct_runtime_state_path(&artifact_root), + &DirectRuntimeState::default(), + ) + .expect("direct runtime state should write"); + + let metadata = collect_live_component_runtime_metadata(&SiteControllerRuntimePlan { + schema: "amber.run.site_controller_runtime_plan".to_string(), + version: 1, + run_id: "run-123".to_string(), + mesh_scope: "amber.test".to_string(), + run_root: temp.path().display().to_string(), + site_id: "direct_local".to_string(), + kind: SiteKind::Direct, + router_identity_id: "/site/direct/router".to_string(), + local_router_control: None, + artifact_dir: artifact_root.display().to_string(), + site_state_root: site_state_root.display().to_string(), + listen_addr: "127.0.0.1:35000".parse().expect("listen addr should parse"), + storage_root: None, + runtime_root: Some(runtime_root.display().to_string()), + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: None, + context: None, + observability_endpoint: None, + launch_env: BTreeMap::new(), + }) + .expect("metadata collection should succeed without a runtime state file"); + + assert!( + metadata.is_empty(), + "static-only sites should not require a site controller runtime state file", + ); + } + + #[test] + fn compose_runtime_metadata_uses_volume_root_control_socket_path() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let artifact_root = temp.path().join("artifact"); + fs::create_dir_all(artifact_root.join(".amber/mesh/provider-net")) + .expect("mesh artifact dir should create"); + + write_json( + &artifact_root.join("mesh-provision-plan.json"), + &MeshProvisionPlan { + version: amber_mesh::MESH_PROVISION_PLAN_VERSION.to_string(), + identity_seed: None, + existing_peer_identities: Vec::new(), + targets: vec![MeshProvisionTarget { + kind: MeshProvisionTargetKind::Component, + config: MeshConfigTemplate { + identity: amber_mesh::MeshIdentityTemplate { + id: "/provider".to_string(), + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:24001".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + output: MeshProvisionOutput::Filesystem { + dir: ".amber/mesh/provider-net".to_string(), + }, + }], + }, + ) + .expect("mesh provision plan should write"); + write_json( + &artifact_root + .join(".amber/mesh/provider-net") + .join(MESH_CONFIG_FILENAME), + &MeshConfigPublic { + identity: MeshIdentityPublic { + id: "/provider".to_string(), + public_key: [9; 32], + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:24001".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + ) + .expect("mesh config should write"); + + let components = collect_compose_artifact_runtime_metadata(&artifact_root, Some("demo")) + .expect("compose metadata should load"); + let provider = components.get("/provider").expect("provider metadata"); + let endpoint = provider + .control_endpoint + .as_ref() + .expect("compose metadata should expose a control endpoint"); + let ControlEndpoint::VolumeSocket { + volume, + socket_path, + } = endpoint + else { + panic!("expected compose control endpoint to use a volume socket"); + }; + assert_eq!(volume, "demo_provider-net-control"); + assert_eq!( + socket_path, "/router-control.sock", + "compose volume sockets must use the volume-root path, not the in-container mount path", + ); + } } fn collect_kubernetes_artifact_runtime_metadata( diff --git a/runtime/site-controller/src/dynamic_caps.rs b/runtime/site-controller/src/dynamic_caps.rs index 3909e26d..4a229e7a 100644 --- a/runtime/site-controller/src/dynamic_caps.rs +++ b/runtime/site-controller/src/dynamic_caps.rs @@ -117,6 +117,16 @@ pub(crate) struct ControlDynamicShareRequest { pub(crate) options: serde_json::Value, } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ControlDynamicGrantAuthoritySyncRequest { + pub(crate) authority_sites: BTreeMap, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ControlDynamicGrantAuthoritySyncResponse { + pub(crate) synced: usize, +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct ControlDynamicInspectRefRequest { pub(crate) holder_component_id: String, @@ -1360,6 +1370,15 @@ pub(crate) fn share_dynamic_capability( }) } +pub(crate) fn sync_dynamic_capability_grant_authority_sites( + state: &mut FrameworkControlState, + authority_sites: &BTreeMap, +) { + state + .dynamic_capability_grant_authority_sites + .extend(authority_sites.clone()); +} + fn caller_has_revoke_authority( state: &FrameworkControlState, roots: &BTreeMap, diff --git a/runtime/site-controller/src/http.rs b/runtime/site-controller/src/http.rs index cb9155e4..75455bed 100644 --- a/runtime/site-controller/src/http.rs +++ b/runtime/site-controller/src/http.rs @@ -12,10 +12,6 @@ use super::{ orchestration::ProtocolApiError, planner::ControlStateApp, state::FRAMEWORK_AUTH_HEADER, *, }; -pub(super) async fn healthz() -> Json { - Json(json!({ "ok": true })) -} - pub(super) async fn cleanup_dynamic_bridge_proxies(app: &ControlStateApp) -> Result<()> { app.runtime.cleanup().await } diff --git a/runtime/site-controller/src/orchestration.rs b/runtime/site-controller/src/orchestration.rs index b8052233..a1d58a43 100644 --- a/runtime/site-controller/src/orchestration.rs +++ b/runtime/site-controller/src/orchestration.rs @@ -289,13 +289,30 @@ async fn peer_router_identity_for_overlay( app: &ControlStateApp, site_id: &str, ) -> std::result::Result { + if let Some(identity) = app.controller_plan.peer_router_identities.get(site_id) { + return Ok(MeshPeer { + id: identity.id.clone(), + public_key: identity.public_key, + }); + } let site_app = SiteControllerApp { control: app.clone(), router_auth_token: app.control_state_auth_token.clone(), + ready: Arc::new(std::sync::atomic::AtomicBool::new(true)), + }; + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + let identity = loop { + match super::site_controller::peer_router_identity_via_router(&site_app, site_id).await { + Ok(identity) => break identity, + Err(err) + if err.0.code == ProtocolErrorCode::ControlStateUnavailable + && tokio::time::Instant::now() < deadline => + { + tokio::time::sleep(Duration::from_millis(250)).await; + } + Err(err) => return Err(err.0), + } }; - let identity = super::site_controller::peer_router_identity_via_router(&site_app, site_id) - .await - .map_err(|err| err.0)?; Ok(MeshPeer { id: identity.id, public_key: identity.public_key, @@ -626,7 +643,7 @@ pub(super) async fn framework_route_overlay_payload( }; for record in state.capability_instances.values() { - let authority_site_id = site_id_for_authority_realm(&state, record.authority_realm_id)?; + let authority_site_id = framework_authority_site_id(&state, record)?; let consumer_site_id = record.recipient_site_id.clone(); if authority_site_id == local_site_id { @@ -655,23 +672,24 @@ pub(super) async fn framework_route_overlay_payload( } let authority_peer = peer_router_identity_for_overlay(app, &authority_site_id).await?; - if !overlay - .peers - .iter() - .any(|peer| peer.id == authority_peer.id) - { - overlay.peers.push(authority_peer.clone()); - } - overlay.inbound_routes.push(framework_component_route( + let route = framework_component_route( record, vec![record.recipient_peer_id.clone()], InboundTarget::MeshForward { peer_addr: peer_router_mesh_addr_for_overlay(app, &authority_site_id)?, - peer_id: authority_peer.id, + peer_id: authority_peer.id.clone(), route_id: record.route_id.clone(), capability: record.capability.clone(), }, - )); + ); + if !overlay + .peers + .iter() + .any(|peer| peer.id == authority_peer.id) + { + overlay.peers.push(authority_peer.clone()); + } + overlay.inbound_routes.push(route); } if overlay.peers.is_empty() && overlay.inbound_routes.is_empty() { @@ -1351,7 +1369,7 @@ fn site_controller_child_needs_prepare(err: &ProtocolErrorResponse, child_id: u6 err.code == ProtocolErrorCode::PublishFailed && err .message - .contains(&format!("site controller child {child_id} is not prepared")) + .contains(&format!("child {child_id} is not prepared")) } pub(super) async fn publish_child_on_site_with_prepare_retry( diff --git a/runtime/site-controller/src/planner.rs b/runtime/site-controller/src/planner.rs index a4061afd..3794c89f 100644 --- a/runtime/site-controller/src/planner.rs +++ b/runtime/site-controller/src/planner.rs @@ -3285,6 +3285,7 @@ pub(super) struct ControlStateApp { pub(super) struct SiteControllerApp { pub(super) control: ControlStateApp, pub(super) router_auth_token: Arc, + pub(super) ready: Arc, } #[derive(Clone)] diff --git a/runtime/site-controller/src/site_controller.rs b/runtime/site-controller/src/site_controller.rs index cf8dd0e0..a1cf83f1 100644 --- a/runtime/site-controller/src/site_controller.rs +++ b/runtime/site-controller/src/site_controller.rs @@ -22,13 +22,14 @@ use super::{ resolve_dynamic_capability_origin_internal, }, dynamic_caps::{ - self, ControlDynamicHeldDetailRequest, ControlDynamicHeldListRequest, + self, ControlDynamicGrantAuthoritySyncRequest, ControlDynamicGrantAuthoritySyncResponse, + ControlDynamicHeldDetailRequest, ControlDynamicHeldListRequest, ControlDynamicInspectRefRequest, ControlDynamicResolveOriginRequest, ControlDynamicRevokeRequest, ControlDynamicShareRequest, InternalDynamicResolveOriginRequest, }, http::{ - authorize_framework_auth_header, cleanup_dynamic_bridge_proxies, healthz, read_json, + authorize_framework_auth_header, cleanup_dynamic_bridge_proxies, read_json, required_header, shutdown_signal, }, orchestration::{ @@ -71,20 +72,40 @@ pub(crate) async fn run_site_controller( authority_locks: Arc::new(Mutex::new(BTreeMap::new())), runtime, }; + let ready = Arc::new(std::sync::atomic::AtomicBool::new(false)); let app_state = SiteControllerApp { control: control.clone(), router_auth_token: Arc::::from(plan.auth_token), + ready: ready.clone(), }; - recover_control_state(&control).await?; let app = site_controller_router(app_state.clone()); let listener = TcpListener::bind(plan.listen_addr) .await .into_diagnostic() .wrap_err_with(|| format!("failed to bind site controller on {}", plan.listen_addr))?; - let serve_result = axum::serve(listener, app.into_make_service()) - .with_graceful_shutdown(shutdown_signal()) + let serve_task = tokio::spawn(async move { + axum::serve(listener, app.into_make_service()) + .with_graceful_shutdown(shutdown_signal()) + .await + .into_diagnostic() + }); + if let Err(err) = recover_control_state(&control).await { + serve_task.abort(); + let _ = serve_task.await; + let cleanup_result = cleanup_dynamic_bridge_proxies(&control).await; + return match cleanup_result { + Ok(()) => Err(err).wrap_err("site controller recovery failed"), + Err(cleanup_err) => Err(miette::miette!( + "site controller recovery failed: {err}\nbridge proxy cleanup failed: \ + {cleanup_err}" + )), + }; + } + ready.store(true, std::sync::atomic::Ordering::SeqCst); + let serve_result = serve_task .await - .into_diagnostic(); + .into_diagnostic() + .wrap_err("site controller task failed")?; let cleanup_result = cleanup_dynamic_bridge_proxies(&control).await; match (serve_result, cleanup_result) { (Ok(()), Ok(())) => Ok(()), @@ -103,8 +124,8 @@ pub(crate) fn site_controller_router(app_state: SiteControllerApp) -> Router { "/v1/controller/dynamic-caps/mcp", super::control_state_mcp::service(app_state.clone()), ) - .route("/", get(healthz)) - .route("/healthz", get(healthz)) + .route("/", get(controller_healthz)) + .route("/healthz", get(controller_healthz)) .route( SITE_CONTROLLER_STATE_PATH, get(get_site_controller_state_route), @@ -166,6 +187,10 @@ pub(crate) fn site_controller_router(app_state: SiteControllerApp) -> Router { "/v1/controller/dynamic-caps/share", post(control_dynamic_share_route), ) + .route( + "/v1/controller/dynamic-caps/grant-authorities/sync", + post(control_dynamic_grant_authorities_sync_route), + ) .route( "/v1/controller/dynamic-caps/inspect-ref", post(control_dynamic_inspect_ref_route), @@ -181,6 +206,16 @@ pub(crate) fn site_controller_router(app_state: SiteControllerApp) -> Router { .with_state(app_state) } +async fn controller_healthz(State(app): State) -> Response { + let ready = app.ready.load(std::sync::atomic::Ordering::SeqCst); + let status = if ready { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + }; + (status, Json(json!({ "ok": ready }))).into_response() +} + async fn get_site_controller_state_route( State(app): State, headers: HeaderMap, @@ -278,6 +313,15 @@ fn controller_local_only(headers: &HeaderMap) -> bool { .is_some_and(|value| value == "1") } +fn ensure_controller_ready(app: &SiteControllerApp) -> std::result::Result<(), ProtocolApiError> { + if app.ready.load(std::sync::atomic::Ordering::SeqCst) { + return Ok(()); + } + Err(ProtocolApiError::control_state_unavailable( + "site controller is still recovering", + )) +} + fn peer_site_router_url_for_site( app: &SiteControllerApp, site_id: &str, @@ -482,15 +526,37 @@ async fn local_share( app: &SiteControllerApp, request: ControlDynamicShareRequest, ) -> std::result::Result { - match control_state_api::execute_dynamic_caps_mutate( + let response = match control_state_api::execute_dynamic_caps_mutate( &app.control, - DynamicCapsMutateRequest::Share(request), + DynamicCapsMutateRequest::Share(request.clone()), ) .await? { - DynamicCapsMutateResponse::Share(response) => Ok(response), + DynamicCapsMutateResponse::Share(response) => response, _ => unreachable!("share should return share response"), + }; + if let Err(err) = sync_shared_grant_authority_site(app, &response).await { + if response.outcome == "created" + && let Some(grant_id) = response.grant_id.as_ref() + && let Err(rollback_err) = control_state_api::execute_dynamic_caps_mutate( + &app.control, + DynamicCapsMutateRequest::Revoke(ControlDynamicRevokeRequest { + caller_component_id: request.caller_component_id, + target: dynamic_caps::DynamicCapabilityControlSourceRequest::Grant { + grant_id: grant_id.clone(), + }, + }), + ) + .await + { + return Err(ProtocolApiError::control_state_unavailable(format!( + "{}; failed to roll back shared grant `{grant_id}` after sync failure: {}", + err.0.message, rollback_err.0.message + ))); + } + return Err(err); } + Ok(response) } async fn local_revoke( @@ -523,6 +589,71 @@ fn site_for_dynamic_source( .map_err(ProtocolApiError::from) } +async fn sync_shared_grant_authority_site( + app: &SiteControllerApp, + response: &amber_mesh::dynamic_caps::ShareResponse, +) -> std::result::Result<(), ProtocolApiError> { + let Some(grant_id) = response.grant_id.as_ref() else { + return Ok(()); + }; + let holder_site_id = { + let state = app.control.control_state.lock().await; + let grant = state + .dynamic_capability_grants + .get(grant_id) + .ok_or_else(|| { + ProtocolApiError::control_state_unavailable(format!( + "dynamic capability share reported grant `{grant_id}` but it is missing from \ + site controller `{}` state", + app.control.controller_plan.site_id + )) + })?; + site_id_for_logical_component(&state, &grant.holder_component_id).map_err(|err| { + ProtocolApiError::control_state_unavailable(format!( + "shared grant `{grant_id}` holder `{}` has no live site assignment: {}", + grant.holder_component_id, err.message + )) + })? + }; + let authority_site_id = app.control.controller_plan.site_id.as_str(); + if holder_site_id == authority_site_id { + return Ok(()); + } + let _: ControlDynamicGrantAuthoritySyncResponse = peer_dynamic_caps_post_via_router( + app, + &holder_site_id, + "/v1/controller/dynamic-caps/grant-authorities/sync", + &ControlDynamicGrantAuthoritySyncRequest { + authority_sites: BTreeMap::from([(grant_id.clone(), authority_site_id.to_string())]), + }, + ) + .await?; + Ok(()) +} + +async fn local_sync_dynamic_grant_authorities( + app: &SiteControllerApp, + request: ControlDynamicGrantAuthoritySyncRequest, +) -> std::result::Result { + let synced = request.authority_sites.len(); + { + let mut state = app.control.control_state.lock().await; + persist_control_state_update( + &mut state, + &app.control.state_path, + "dynamic capability grant authority sync", + |state| { + dynamic_caps::sync_dynamic_capability_grant_authority_sites( + state, + &request.authority_sites, + ); + Ok(()) + }, + )?; + } + Ok(ControlDynamicGrantAuthoritySyncResponse { synced }) +} + fn site_for_held_id( state: &FrameworkControlState, held_id: &str, @@ -554,10 +685,7 @@ fn framework_request_site_id( state: &FrameworkControlState, record: &CapabilityInstanceRecord, ) -> std::result::Result { - match site_id_for_authority_realm(state, record.authority_realm_id) { - Ok(site_id) => Ok(site_id), - Err(_) => Ok(record.recipient_site_id.clone()), - } + framework_authority_site_id(state, record).map_err(ProtocolApiError::from) } pub(super) async fn execute_site_controller_framework_inspect( @@ -566,6 +694,7 @@ pub(super) async fn execute_site_controller_framework_inspect( state: &FrameworkControlState, request: FrameworkComponentInspectRequest, ) -> std::result::Result { + ensure_controller_ready(app)?; let authority_site = framework_request_site_id(state, record)?; if authority_site != app.control.controller_plan.site_id { return Err(ProtocolApiError::control_state_unavailable(format!( @@ -586,6 +715,7 @@ pub(super) async fn execute_site_controller_framework_mutate( state: &FrameworkControlState, request: ccs_api::FrameworkComponentMutateRequest, ) -> std::result::Result { + ensure_controller_ready(app)?; let authority_site = framework_request_site_id(state, record)?; if authority_site != app.control.controller_plan.site_id { return Err(ProtocolApiError::control_state_unavailable(format!( @@ -618,6 +748,7 @@ pub(super) async fn execute_site_controller_dynamic_caps_inspect( request: DynamicCapsInspectRequest, local_only: bool, ) -> std::result::Result { + ensure_controller_ready(app)?; match request { DynamicCapsInspectRequest::HeldList(request) => { if local_only { @@ -730,6 +861,7 @@ pub(super) async fn execute_site_controller_dynamic_caps_mutate( request: DynamicCapsMutateRequest, local_only: bool, ) -> std::result::Result { + ensure_controller_ready(app)?; match request { DynamicCapsMutateRequest::Share(request) => { if local_only { @@ -1082,6 +1214,17 @@ async fn control_dynamic_share_route( } } +async fn control_dynamic_grant_authorities_sync_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + Ok(Json( + local_sync_dynamic_grant_authorities(&app, request).await?, + )) +} + async fn control_dynamic_inspect_ref_route( State(app): State, headers: HeaderMap, diff --git a/runtime/site-controller/src/state.rs b/runtime/site-controller/src/state.rs index ff2b760d..79272bad 100644 --- a/runtime/site-controller/src/state.rs +++ b/runtime/site-controller/src/state.rs @@ -236,6 +236,8 @@ pub struct FrameworkControlState { pub(crate) next_dynamic_capability_grant_id: u64, #[serde(default)] pub(crate) dynamic_capability_grants: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub(crate) dynamic_capability_grant_authority_sites: BTreeMap, #[serde(default)] pub(crate) dynamic_capability_journal: Vec, #[serde(default)] @@ -264,6 +266,8 @@ pub struct SiteControllerPlan { #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub peer_site_router_urls: BTreeMap, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub peer_router_identities: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub peer_router_mesh_addrs: BTreeMap, #[serde(default, skip_serializing_if = "Option::is_none")] pub local_router_control: Option, @@ -450,6 +454,7 @@ fn build_control_state_with_signing_seed( dynamic_capability_signing_seed_b64: dynamic_capability_signing_seed_b64.to_string(), next_dynamic_capability_grant_id: 0, dynamic_capability_grants: BTreeMap::new(), + dynamic_capability_grant_authority_sites: BTreeMap::new(), dynamic_capability_journal: Vec::new(), live_children: Vec::new(), pending_creates: Vec::new(), @@ -498,6 +503,20 @@ pub(crate) fn localize_framework_control_state( let roots = dynamic_caps::derive_root_authorities(state) .map_err(|err| miette::miette!(err.message.clone()))?; + let local_grant_authority_sites = state + .dynamic_capability_grants + .iter() + .filter_map(|(grant_id, grant)| { + let authority_site_id = roots + .get(&dynamic_caps::root_authority_key( + &grant.root_authority_selector, + )) + .and_then(|root| component_site_id(state, &root.holder_component_id).ok())?; + let holder_site_id = component_site_id(state, &grant.holder_component_id).ok()?; + (holder_site_id == site_id && authority_site_id != site_id) + .then_some((grant_id.clone(), authority_site_id)) + }) + .collect::>(); let local_grant_ids = state .dynamic_capability_grants .iter() @@ -514,6 +533,7 @@ pub(crate) fn localize_framework_control_state( state .dynamic_capability_grants .retain(|grant_id, _| local_grant_ids.contains(grant_id)); + state.dynamic_capability_grant_authority_sites = local_grant_authority_sites; state.dynamic_capability_journal.retain(|entry| { entry .grant_id @@ -536,6 +556,7 @@ fn child_authority_site_id( &authority_moniker, &format!("authority realm `{authority_moniker}`"), ) + .or_else(|_| child_runtime_site_id(child)) } fn component_site_id( @@ -556,12 +577,12 @@ fn site_id_for_moniker( moniker: &str, subject: &str, ) -> std::result::Result { - let assignments = live_assignment_map(state); - if let Some(site_id) = assignments.get(moniker) { + let components = planned_component_site_map(state); + if let Some(site_id) = components.get(moniker) { return Ok(site_id.clone()); } - let descendant_sites = assignments + let descendant_sites = components .iter() .filter(|(assigned_moniker, _)| moniker_contains(assigned_moniker, moniker)) .map(|(_, site_id)| site_id.clone()) @@ -586,6 +607,15 @@ fn site_id_for_moniker( )) } +fn planned_component_site_map(state: &FrameworkControlState) -> BTreeMap { + let mut components = state.placement.placement_components.clone(); + components.extend(state.placement.assignments.clone()); + for child in visible_child_records(state) { + components.extend(child.assignments.clone()); + } + components +} + fn moniker_contains(candidate: &str, realm: &str) -> bool { if realm == "/" { return true; @@ -1204,6 +1234,7 @@ pub fn write_site_controller_plan( authority_url: &str, router_identity_id: &str, peer_site_router_urls: &BTreeMap, + peer_router_identities: &BTreeMap, peer_router_mesh_addrs: &BTreeMap, local_router_control: Option<&str>, published_router_mesh_addr: Option<&str>, @@ -1234,6 +1265,7 @@ pub fn write_site_controller_plan( authority_url: authority_url.to_string(), router_identity_id: router_identity_id.to_string(), peer_site_router_urls: peer_site_router_urls.clone(), + peer_router_identities: peer_router_identities.clone(), peer_router_mesh_addrs: peer_router_mesh_addrs.clone(), local_router_control: local_router_control.map(str::to_string), published_router_mesh_addr: published_router_mesh_addr.map(str::to_string), @@ -1272,28 +1304,74 @@ pub(crate) fn site_id_for_authority_realm( ) } +pub(crate) fn framework_authority_site_id( + state: &FrameworkControlState, + record: &CapabilityInstanceRecord, +) -> std::result::Result { + if let Ok(site_id) = site_id_for_authority_realm(state, record.authority_realm_id) { + return Ok(site_id); + } + let authority = record.authority_realm_moniker.as_str(); + let recipient = record.recipient_component_moniker.as_str(); + if authority != "/" + && recipient != authority + && !recipient.starts_with(&format!("{authority}/")) + { + return Err(protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "framework.component recipient `{recipient}` is not inside authority realm \ + `{authority}`" + ), + )); + } + let assignments = planned_component_site_map(state); + let recipient_segments = recipient + .trim_start_matches('/') + .split('/') + .filter(|segment| !segment.is_empty()) + .collect::>(); + let authority_depth = authority + .trim_matches('/') + .split('/') + .filter(|segment| !segment.is_empty()) + .count(); + for depth in authority_depth..=recipient_segments.len() { + let moniker = if depth == 0 { + "/".to_string() + } else { + format!("/{}", recipient_segments[..depth].join("/")) + }; + if let Some(site_id) = assignments.get(&moniker) { + return Ok(site_id.clone()); + } + } + Err(protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "framework.component authority realm `{authority}` has no assigned component on the \ + path to recipient `{recipient}`" + ), + )) +} + pub(crate) fn site_id_for_dynamic_grant( state: &FrameworkControlState, grant_id: &str, ) -> std::result::Result { - let grant = state - .dynamic_capability_grants + if let Some(grant) = state.dynamic_capability_grants.get(grant_id) { + return site_id_for_root_authority_selector(state, &grant.root_authority_selector); + } + state + .dynamic_capability_grant_authority_sites .get(grant_id) + .cloned() .ok_or_else(|| { protocol_error( ProtocolErrorCode::UnknownSource, &format!("dynamic grant `{grant_id}` is not live"), ) - })?; - site_id_for_logical_component(state, &grant.holder_component_id).map_err(|_| { - protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &format!( - "dynamic grant `{grant_id}` holder `{}` is missing a live site assignment", - grant.holder_component_id - ), - ) - }) + }) } pub(crate) fn site_id_for_logical_component( diff --git a/runtime/site-controller/src/tests.rs b/runtime/site-controller/src/tests.rs index 63bc79f2..19177e97 100644 --- a/runtime/site-controller/src/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -51,6 +51,10 @@ fn direct_runtime_state_path(plan_root: &Path) -> PathBuf { #[derive(Clone, Default)] struct TestSiteControllerRuntime; +fn ready_site_controller_flag() -> Arc { + Arc::new(std::sync::atomic::AtomicBool::new(true)) +} + impl SiteControllerRuntime for TestSiteControllerRuntime { fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) @@ -1927,6 +1931,142 @@ async fn dynamic_grant_routes_to_holder_site_not_offered_site_order() { ); } +#[tokio::test] +async fn dynamic_caps_cross_site_share_syncs_holder_authority_through_site_router() { + let dir = TempDir::new().expect("temp dir"); + let base = compile_dynamic_caps_binding_state().await; + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "direct_a".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + ), + ( + "direct_b".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_a".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([ + ("/provider".to_string(), "direct_a".to_string()), + ("/alice".to_string(), "direct_a".to_string()), + ("/bob".to_string(), "direct_b".to_string()), + ]), + dynamic_capabilities: None, + framework_children: None, + }; + let mut authority_state = compile_control_state_from_ir_with_run_id( + base.base_scenario.clone(), + Some(&placement), + "test-run", + ) + .await; + localize_framework_control_state(&mut authority_state, "direct_a") + .expect("authority site state should localize"); + let alice_root = super::dynamic_caps::source_key_from_held_id( + &authority_state, + "components./alice", + &root_held_id_for(&authority_state, "components./alice"), + ) + .expect("alice root source should resolve"); + let dynamic_caps::DynamicCapabilitySourceKey::RootAuthority(root_authority_selector) = + alice_root + else { + panic!("alice root held id should resolve to a root authority selector"); + }; + let state_path = dir.path().join("control-state.json"); + write_control_state(&state_path, &authority_state).expect("authority state should write"); + + let sync_requests = Arc::new(std::sync::Mutex::new(Vec::< + dynamic_caps::ControlDynamicGrantAuthoritySyncRequest, + >::new())); + let router = Router::new().route( + "/v1/controller/dynamic-caps/grant-authorities/sync", + axum::routing::post({ + let sync_requests = sync_requests.clone(); + move |headers: HeaderMap, + Json(request): Json| { + let sync_requests = sync_requests.clone(); + async move { + assert_eq!( + headers + .get(super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER) + .and_then(|value| value.to_str().ok()), + Some("1"), + "cross-site share sync should stay local on the destination controller", + ); + sync_requests + .lock() + .expect("sync request log poisoned") + .push(request.clone()); + Json(dynamic_caps::ControlDynamicGrantAuthoritySyncResponse { + synced: request.authority_sites.len(), + }) + } + } + }), + ); + let (holder_base_url, _holder_handle) = spawn_test_router(router).await; + + let mut app = test_control_state_app(&dir, authority_state, state_path); + let controller_plan = Arc::make_mut(&mut app.controller_plan); + controller_plan.site_id = "direct_a".to_string(); + controller_plan.router_identity_id = "/site/direct_a/router".to_string(); + controller_plan.peer_site_router_urls = + BTreeMap::from([("direct_b".to_string(), holder_base_url)]); + let controller_app = SiteControllerApp { + control: app, + router_auth_token: Arc::::from("test-router-auth"), + ready: ready_site_controller_flag(), + }; + + let response = super::site_controller::execute_site_controller_dynamic_caps_mutate( + &controller_app, + super::control_state_api::DynamicCapsMutateRequest::Share( + dynamic_caps::ControlDynamicShareRequest { + caller_component_id: "components./alice".to_string(), + source: dynamic_caps::DynamicCapabilityControlSourceRequest::RootAuthority { + root_authority_selector, + }, + recipient_component_id: "components./bob".to_string(), + idempotency_key: None, + options: json!({}), + }, + ), + false, + ) + .await + .expect("authority site should create the share and sync it through the holder router"); + + let super::control_state_api::DynamicCapsMutateResponse::Share(response) = response else { + panic!("share should return a share response"); + }; + let grant_id = response + .grant_id + .clone() + .expect("cross-site share should create a concrete grant id"); + assert_eq!(response.outcome, "created"); + + let sync_requests = sync_requests.lock().expect("sync request log poisoned"); + assert_eq!(sync_requests.len(), 1, "expected exactly one holder sync"); + assert_eq!( + sync_requests[0].authority_sites, + BTreeMap::from([(grant_id, "direct_a".to_string())]), + "the holder site should learn that direct_a remains authoritative for the shared grant", + ); +} + async fn compile_empty_control_state() -> (TempDir, FrameworkControlState, PathBuf) { let dir = TempDir::new().expect("temp dir"); let root_path = dir.path().join("root.json5"); @@ -2303,6 +2443,7 @@ fn test_control_state_app( authority_url: "http://127.0.0.1:0".to_string(), router_identity_id: format!("/site/{site_id}/router"), peer_site_router_urls: BTreeMap::new(), + peer_router_identities: BTreeMap::new(), peer_router_mesh_addrs: BTreeMap::new(), local_router_control: None, published_router_mesh_addr: Some("127.0.0.1:24000".to_string()), @@ -2478,7 +2619,7 @@ fn normalize_dynamic_share_ref(value: &mut Value) { } const TEST_SITE_STATE_SCHEMA: &str = "amber.run.site_state"; -const TEST_SITE_STATE_VERSION: u32 = 2; +const TEST_SITE_STATE_VERSION: u32 = 3; const TEST_REQUEST_TIMEOUT: StdDuration = StdDuration::from_secs(15); async fn with_test_timeout(label: impl Into, future: impl Future) -> T { @@ -2827,6 +2968,7 @@ impl FrameworkMcpHarness { let controller_app = SiteControllerApp { control: app, router_auth_token: Arc::::from(auth_token.clone()), + ready: ready_site_controller_flag(), }; handles.push(tokio::spawn(async move { axum::serve( @@ -3115,6 +3257,7 @@ async fn framework_component_rejects_stale_nonlocal_controller_delivery() { runtime: app.runtime.clone(), }, router_auth_token: Arc::::from("test-router-auth"), + ready: ready_site_controller_flag(), }; let err = match super::site_controller::execute_site_controller_framework_inspect( @@ -3138,6 +3281,31 @@ async fn framework_component_rejects_stale_nonlocal_controller_delivery() { ); } +#[tokio::test] +async fn framework_component_rejects_requests_while_controller_recovers() { + let (_dir, state, state_path, record) = compile_framework_binding_control_state().await; + let controller_app = SiteControllerApp { + control: test_control_state_app(&_dir, state.clone(), state_path), + router_auth_token: Arc::::from("test-router-auth"), + ready: Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let err = match super::site_controller::execute_site_controller_framework_inspect( + &controller_app, + &record, + &state, + FrameworkComponentInspectRequest::ListTemplates, + ) + .await + { + Ok(_) => panic!("recovering controller should reject public framework requests"), + Err(err) => err, + }; + + assert_eq!(err.0.code, ProtocolErrorCode::ControlStateUnavailable); + assert_eq!(err.0.message, "site controller is still recovering"); +} + #[tokio::test] async fn framework_component_cross_site_routes_forward_through_site_routers() { let (dir, mut state, _state_path, _) = compile_framework_binding_control_state().await; @@ -3278,6 +3446,205 @@ async fn framework_component_cross_site_routes_forward_through_site_routers() { ); } +#[tokio::test] +async fn framework_route_overlay_payload_uses_planned_peer_router_identities() { + let (dir, mut state, state_path, _) = compile_framework_binding_control_state().await; + state.placement.offered_sites = BTreeMap::from([ + ( + "authority".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + ), + ( + "consumer".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + ), + ]); + state.placement.defaults = PlacementDefaults { + path: Some("authority".to_string()), + ..PlacementDefaults::default() + }; + state.placement.placement_components = BTreeMap::from([ + ("/".to_string(), "authority".to_string()), + ("/admin".to_string(), "consumer".to_string()), + ]); + state.placement.assignments = state.placement.placement_components.clone(); + refresh_capability_instances(&mut state).expect("framework routes should refresh"); + write_control_state(&state_path, &state).expect("state should write"); + let app = test_control_state_app(&dir, state, state_path); + + let mut consumer_plan = app.controller_plan.as_ref().clone(); + consumer_plan.site_id = "consumer".to_string(); + consumer_plan.router_identity_id = "/site/consumer/router".to_string(); + consumer_plan.peer_router_identities = BTreeMap::from([( + "authority".to_string(), + MeshIdentityPublic { + id: "/site/authority/router".to_string(), + public_key: [9u8; 32], + mesh_scope: Some("test-mesh".to_string()), + }, + )]); + consumer_plan.peer_router_mesh_addrs = + BTreeMap::from([("authority".to_string(), "127.0.0.1:24000".to_string())]); + let consumer_app = ControlStateApp { + controller_plan: Arc::new(consumer_plan), + ..app + }; + + let overlay = framework_route_overlay_payload(&consumer_app) + .await + .expect("consumer framework routes should materialize") + .expect("consumer site should get a framework route overlay"); + + assert!( + overlay + .peers + .iter() + .any(|peer| peer.id == "/site/authority/router" && peer.public_key == [9u8; 32]), + "consumer overlay should use the planned authority router identity without a peer \ + controller round trip", + ); +} + +#[tokio::test] +async fn recover_control_state_reconciles_framework_routes_without_live_peer_controllers() { + let (dir, mut state, state_path, _) = compile_framework_binding_control_state().await; + state.placement.offered_sites = BTreeMap::from([ + ( + "authority".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + ), + ( + "consumer".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + ), + ]); + state.placement.defaults = PlacementDefaults { + path: Some("authority".to_string()), + ..PlacementDefaults::default() + }; + state.placement.placement_components = BTreeMap::from([ + ("/".to_string(), "authority".to_string()), + ("/admin".to_string(), "consumer".to_string()), + ]); + state.placement.assignments = state.placement.placement_components.clone(); + refresh_capability_instances(&mut state).expect("framework routes should refresh"); + write_control_state(&state_path, &state).expect("state should write"); + let app = test_control_state_app(&dir, state, state_path); + + let mut authority_plan = app.controller_plan.as_ref().clone(); + authority_plan.site_id = "authority".to_string(); + authority_plan.router_identity_id = "/site/authority/router".to_string(); + authority_plan.peer_router_identities = BTreeMap::from([( + "consumer".to_string(), + MeshIdentityPublic { + id: "/site/consumer/router".to_string(), + public_key: [8u8; 32], + mesh_scope: Some("test-mesh".to_string()), + }, + )]); + let authority_app = ControlStateApp { + controller_plan: Arc::new(authority_plan), + ..app + }; + + recover_control_state(&authority_app).await.expect( + "recovery should not require a live peer controller when peer router identities were \ + already planned", + ); +} + +#[test] +fn inject_site_controller_peer_router_routes_records_peer_router_identities() { + let temp = TempDir::new().expect("tempdir"); + let artifact_root = temp.path(); + write_json( + &artifact_root.join("mesh-provision-plan.json"), + &amber_mesh::MeshProvisionPlan { + version: amber_mesh::MESH_PROVISION_PLAN_VERSION.to_string(), + identity_seed: None, + existing_peer_identities: Vec::new(), + targets: vec![amber_mesh::MeshProvisionTarget { + kind: amber_mesh::MeshProvisionTargetKind::Router, + config: amber_mesh::MeshConfigTemplate { + identity: amber_mesh::MeshIdentityTemplate { + id: "/site/local/router".to_string(), + mesh_scope: Some("test-mesh".to_string()), + }, + mesh_listen: "127.0.0.1:24000".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: TransportConfig::NoiseIk {}, + }, + output: amber_mesh::MeshProvisionOutput::Filesystem { + dir: "mesh/router".to_string(), + }, + }], + }, + ) + .expect("mesh provision plan should write"); + + inject_site_controller_peer_router_routes( + artifact_root, + "local", + &["/site/peer/router".to_string()], + &[SiteControllerPeerRouterRoute { + site_id: "peer".to_string(), + peer_router: MeshIdentityPublic { + id: "/site/peer/router".to_string(), + public_key: [5u8; 32], + mesh_scope: Some("test-mesh".to_string()), + }, + peer_addr: "10.0.0.2:24000".to_string(), + listen_addr: "127.0.0.1".to_string(), + listen_port: 24123, + }], + ) + .expect("site controller peer routes should inject"); + + let plan: amber_mesh::MeshProvisionPlan = read_json( + &artifact_root.join("mesh-provision-plan.json"), + "mesh provision plan", + ) + .expect("mesh provision plan should read"); + assert!( + plan.existing_peer_identities + .iter() + .any(|identity| identity.id == "/site/peer/router" && identity.public_key == [5u8; 32]), + "the injected router plan should carry peer router identities so startup does not need \ + runtime discovery", + ); + let router = plan + .targets + .iter() + .find(|target| matches!(target.kind, amber_mesh::MeshProvisionTargetKind::Router)) + .expect("router target should remain present"); + assert!( + router + .config + .outbound + .iter() + .any(|route| route.route_id == "site-controller:peer" + && route.peer_id == "/site/peer/router"), + "the router plan should include the site-controller forwarding route for the peer site", + ); +} + #[tokio::test] async fn prepare_child_on_site_rejects_nonlocal_site_plan() { let dir = TempDir::new().expect("temp dir"); @@ -3361,6 +3728,7 @@ impl DynamicCapsMcpHarness { let controller_app = SiteControllerApp { control: app, router_auth_token: Arc::::from("test-router-auth"), + ready: ready_site_controller_flag(), }; handles.push(tokio::spawn(async move { axum::serve( @@ -3601,6 +3969,7 @@ async fn dynamic_caps_held_list_ignores_unrouted_offered_sites() { let controller_app = SiteControllerApp { control: test_control_state_app(&dir, state, state_path), router_auth_token: Arc::::from("test-router-auth"), + ready: ready_site_controller_flag(), }; let response = super::site_controller::execute_site_controller_dynamic_caps_inspect( @@ -3626,6 +3995,229 @@ async fn dynamic_caps_held_list_ignores_unrouted_offered_sites() { ); } +#[tokio::test] +async fn localize_framework_control_state_tracks_remote_grant_authority_sites() { + let base = compile_dynamic_caps_binding_state().await; + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "direct_a".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + ), + ( + "direct_b".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_a".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([ + ("/provider".to_string(), "direct_a".to_string()), + ("/alice".to_string(), "direct_a".to_string()), + ("/bob".to_string(), "direct_b".to_string()), + ]), + dynamic_capabilities: None, + framework_children: None, + }; + let mut state = compile_control_state_from_ir_with_run_id( + base.base_scenario.clone(), + Some(&placement), + "test-run", + ) + .await; + let alice_root = super::dynamic_caps::source_key_from_held_id( + &state, + "components./alice", + &root_held_id_for(&state, "components./alice"), + ) + .expect("alice root source should resolve"); + let share = super::dynamic_caps::share_dynamic_capability( + &mut state, + "components./alice", + &alice_root, + "components./bob", + None, + &json!({}), + ) + .expect("cross-site share should succeed"); + let super::dynamic_caps::DynamicCapabilityShareOutcome::Created { grant_id, .. } = share else { + panic!("cross-site share should create a grant"); + }; + + localize_framework_control_state(&mut state, "direct_b") + .expect("recipient site state should localize"); + + assert!( + state.dynamic_capability_grants.is_empty(), + "recipient site should not retain the authoritative grant record", + ); + assert_eq!( + state + .dynamic_capability_grant_authority_sites + .get(&grant_id), + Some(&"direct_a".to_string()), + "recipient site should retain a lightweight authority index for delegated grants", + ); +} + +#[tokio::test] +async fn inspect_ref_routes_remote_grants_via_synced_authority_site() { + let dir = TempDir::new().expect("temp dir"); + let base = compile_dynamic_caps_binding_state().await; + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "direct_a".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + ), + ( + "direct_b".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_a".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([ + ("/provider".to_string(), "direct_a".to_string()), + ("/alice".to_string(), "direct_a".to_string()), + ("/bob".to_string(), "direct_b".to_string()), + ]), + dynamic_capabilities: None, + framework_children: None, + }; + let mut authoritative = compile_control_state_from_ir_with_run_id( + base.base_scenario.clone(), + Some(&placement), + "test-run", + ) + .await; + let alice_root = super::dynamic_caps::source_key_from_held_id( + &authoritative, + "components./alice", + &root_held_id_for(&authoritative, "components./alice"), + ) + .expect("alice root source should resolve"); + let share = super::dynamic_caps::share_dynamic_capability( + &mut authoritative, + "components./alice", + &alice_root, + "components./bob", + None, + &json!({}), + ) + .expect("cross-site share should succeed"); + let (grant_id, shared_ref) = match share { + super::dynamic_caps::DynamicCapabilityShareOutcome::Created { grant_id, r#ref } => { + (grant_id, r#ref) + } + _ => panic!("cross-site share should create a grant"), + }; + + let mut holder_state = authoritative.clone(); + localize_framework_control_state(&mut holder_state, "direct_b") + .expect("holder site state should localize"); + let state_path = dir.path().join("control-state.json"); + write_control_state(&state_path, &holder_state).expect("holder state should write"); + + let hits = Arc::new(std::sync::Mutex::new(Vec::::new())); + let remote_grant_id = grant_id.clone(); + let router = Router::new().route( + "/v1/controller/dynamic-caps/inspect-ref", + axum::routing::post({ + let hits = hits.clone(); + let remote_grant_id = remote_grant_id.clone(); + move |headers: HeaderMap, + Json(request): Json| { + let hits = hits.clone(); + let remote_grant_id = remote_grant_id.clone(); + async move { + assert_eq!( + headers + .get(super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER) + .and_then(|value| value.to_str().ok()), + Some("1"), + "peer-routed inspect_ref should stay local on the destination controller", + ); + hits.lock() + .expect("inspect-ref hit log poisoned") + .push(request.r#ref.clone()); + Json(amber_mesh::dynamic_caps::InspectRefResponse { + state: HeldEntryState::Live, + grant_id: remote_grant_id, + holder_component_id: "components./bob".to_string(), + descriptor: DescriptorIr { + kind: "http".to_string(), + label: "provider.http".to_string(), + profile: None, + }, + held_id: Some(super::dynamic_caps::held_id_for_grant("g_0000000000000000")), + }) + } + } + }), + ); + let (authority_base_url, _authority_handle) = spawn_test_router(router).await; + + let mut app = test_control_state_app(&dir, holder_state, state_path); + let controller_plan = Arc::make_mut(&mut app.controller_plan); + controller_plan.site_id = "direct_b".to_string(); + controller_plan.router_identity_id = "/site/direct_b/router".to_string(); + controller_plan.peer_site_router_urls = + BTreeMap::from([("direct_a".to_string(), authority_base_url)]); + let controller_app = SiteControllerApp { + control: app, + router_auth_token: Arc::::from("test-router-auth"), + ready: ready_site_controller_flag(), + }; + + let response = super::site_controller::execute_site_controller_dynamic_caps_inspect( + &controller_app, + super::control_state_api::DynamicCapsInspectRequest::InspectRef( + dynamic_caps::ControlDynamicInspectRefRequest { + holder_component_id: "components./bob".to_string(), + r#ref: shared_ref.clone(), + }, + ), + false, + ) + .await + .expect("holder site should route inspect_ref through the authority site"); + + let super::control_state_api::DynamicCapsInspectResponse::InspectRef(response) = response + else { + panic!("inspect_ref should return inspect response"); + }; + assert_eq!(response.state, HeldEntryState::Live); + assert_eq!(response.grant_id, grant_id); + assert_eq!( + hits.lock() + .expect("inspect-ref hit log poisoned") + .as_slice(), + &[shared_ref], + "inspect_ref should route exactly once through the authority site router", + ); +} + #[tokio::test] async fn dynamic_caps_mcp_discovers_compact_surface() { let harness = DynamicCapsMcpHarness::start().await; @@ -3903,6 +4495,68 @@ async fn dynamic_caps_mcp_matches_http_surface() { ); } +#[tokio::test] +async fn dynamic_caps_resolve_origin_tolerates_missing_static_runtime_state_file() { + let harness = DynamicCapsMcpHarness::start().await; + + let held: amber_mesh::dynamic_caps::HeldListResponse = harness + .post_json( + "/v1/controller/dynamic-caps/held", + &dynamic_caps::ControlDynamicHeldListRequest { + holder_component_id: "components./alice".to_string(), + }, + ) + .await; + let root_held_id = held + .held + .iter() + .find(|entry| entry.entry_kind == HeldEntryKind::RootAuthority) + .map(|entry| entry.held_id.clone()) + .expect("alice should have a root authority"); + let detail: HeldEntryDetail = harness + .post_json( + "/v1/controller/dynamic-caps/held/detail", + &dynamic_caps::ControlDynamicHeldDetailRequest { + holder_component_id: "components./alice".to_string(), + held_id: root_held_id, + }, + ) + .await; + let selector = detail + .summary + .root_authority_selector + .clone() + .expect("root held detail should include a root authority selector"); + + fs::remove_file( + harness + ._dir + .path() + .join("state") + .join("direct_local") + .join("site-controller-runtime-state.json"), + ) + .expect("fixture runtime state should be removable"); + + let resolve_origin: dynamic_caps::ControlDynamicResolveOriginResponse = harness + .post_json( + "/v1/controller/dynamic-caps/resolve-origin", + &dynamic_caps::ControlDynamicResolveOriginRequest { + holder_component_id: "components./alice".to_string(), + source: dynamic_caps::DynamicCapabilityControlSourceRequest::RootAuthority { + root_authority_selector: selector, + }, + }, + ) + .await; + + assert!( + !resolve_origin.origin_peer_addr.is_empty(), + "static components should still resolve a live origin even when the dynamic runtime state \ + file is absent", + ); +} + #[tokio::test] async fn create_snapshot_and_destroy_exact_child() { let dir = TempDir::new().expect("temp dir"); @@ -4713,6 +5367,324 @@ async fn dynamic_framework_bindings_refresh_capability_instances_and_preserve_or ); } +#[tokio::test] +async fn delegated_cross_site_framework_requests_route_to_the_forwarded_authority_site() { + let dir = TempDir::new().expect("temp dir"); + let root_path = dir.path().join("root.json5"); + let parent_path = dir.path().join("parent.json5"); + let worker_path = dir.path().join("worker.json5"); + let root_worker_path = dir.path().join("root-worker.json5"); + write_file( + &worker_path, + r#" + { + manifest_version: "0.3.0", + slots: { + realm: { kind: "component", optional: true } + }, + program: { + path: "/bin/echo", + args: ["worker", "${slots.realm.url}"], + network: { endpoints: [{ name: "http", port: 8080 }] } + }, + provides: { http: { kind: "http", endpoint: "http" } }, + exports: { http: "provides.http" }, + } + "#, + ); + write_file( + &root_worker_path, + r#" + { + manifest_version: "0.3.0", + program: { + path: "/bin/echo", + args: ["root-worker"], + network: { endpoints: [{ name: "http", port: 8082 }] } + }, + provides: { http: { kind: "http", endpoint: "http" } }, + exports: { http: "provides.http" }, + } + "#, + ); + write_file( + &parent_path, + &format!( + r#" + {{ + manifest_version: "0.3.0", + slots: {{ + realm: {{ kind: "component", optional: true }} + }}, + program: {{ + path: "/bin/echo", + args: ["parent", "${{slots.realm.url}}"] + }}, + child_templates: {{ + worker: {{ + manifest: "{worker}", + bindings: {{ + realm: "slots.realm" + }} + }} + }}, + }} + "#, + worker = file_url(&worker_path), + ), + ); + write_file( + &root_path, + &format!( + r##" + {{ + manifest_version: "0.3.0", + slots: {{ + realm: {{ kind: "component", optional: true }} + }}, + child_templates: {{ + root_worker: {{ + manifest: "{root_worker}" + }} + }}, + components: {{ + parent: "{parent}" + }}, + bindings: [ + {{ to: "#parent.realm", from: "framework.component" }} + ], + }} + "##, + root_worker = file_url(&root_worker_path), + parent = file_url(&parent_path), + ), + ); + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "compose_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + ), + ( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_local".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([ + ("/parent".to_string(), "compose_local".to_string()), + ("/parent/delegate".to_string(), "direct_local".to_string()), + ("/sibling".to_string(), "compose_local".to_string()), + ]), + dynamic_capabilities: None, + framework_children: None, + }; + + let mut authoritative = + compile_control_state_with_placement(&root_path, Some(&placement)).await; + let authoritative_state_path = dir.path().join("authoritative-state.json"); + write_control_state(&authoritative_state_path, &authoritative).expect("state should write"); + let base = Scenario::try_from(authoritative.base_scenario.clone()).expect("base scenario"); + let parent_id = base + .components_iter() + .find(|(_, component)| component.moniker.as_str() == "/parent") + .map(|(id, _)| id.0) + .expect("parent component should exist"); + create_child( + &mut authoritative, + parent_id, + CreateChildRequest { + template: "worker".to_string(), + name: "delegate".to_string(), + manifest: None, + config: BTreeMap::new(), + bindings: BTreeMap::new(), + }, + &authoritative_state_path, + ) + .await + .expect("delegate child should be created"); + + let mut direct_state = authoritative.clone(); + localize_framework_control_state(&mut direct_state, "direct_local") + .expect("direct-local state should localize"); + direct_state.placement.assignments = + BTreeMap::from([("/parent/delegate".to_string(), "direct_local".to_string())]); + let direct_record = direct_state + .capability_instances + .values() + .find(|record| record.recipient_component_moniker == "/parent/delegate") + .cloned() + .expect("delegate should have a forwarded framework capability instance"); + assert_eq!( + framework_authority_site_id(&direct_state, &direct_record) + .expect("authority site should resolve through composite placement"), + "compose_local", + "delegated root authority should stay anchored at the forwarded realm site", + ); + let router_public_key_b64 = base64::engine::general_purpose::STANDARD.encode([11u8; 32]); + let peer_router_attempts = std::sync::Arc::new(std::sync::Mutex::new(0usize)); + let peer_router = Router::new().route( + "/v1/controller/router-identity", + axum::routing::get({ + let router_public_key_b64 = router_public_key_b64.clone(); + let peer_router_attempts = peer_router_attempts.clone(); + move || { + let router_public_key_b64 = router_public_key_b64.clone(); + let peer_router_attempts = peer_router_attempts.clone(); + async move { + let mut attempts = peer_router_attempts + .lock() + .expect("peer router attempt log poisoned"); + *attempts += 1; + if *attempts == 1 { + return Err(StatusCode::BAD_GATEWAY); + } + Ok(Json(RouterIdentityResponse { + id: "/site/direct_local/router".to_string(), + public_key_b64: router_public_key_b64, + })) + } + } + }), + ); + let (peer_router_base_url, _peer_router_handle) = spawn_test_router(peer_router).await; + let direct_state_path = dir.path().join("direct-state.json"); + write_control_state(&direct_state_path, &direct_state).expect("direct state should write"); + let mut direct_plan = test_control_state_app(&dir, direct_state.clone(), direct_state_path) + .controller_plan + .as_ref() + .clone(); + direct_plan.site_id = "direct_local".to_string(); + direct_plan.kind = SiteKind::Direct; + direct_plan.peer_site_router_urls = + BTreeMap::from([("compose_local".to_string(), peer_router_base_url.clone())]); + direct_plan.peer_router_mesh_addrs = + BTreeMap::from([("compose_local".to_string(), "127.0.0.1:25000".to_string())]); + let direct_controller_app = SiteControllerApp { + control: ControlStateApp { + control_state: Arc::new(Mutex::new(direct_state.clone())), + controller_plan: Arc::new(direct_plan), + ..test_control_state_app( + &dir, + direct_state.clone(), + dir.path().join("direct-state.json"), + ) + }, + router_auth_token: Arc::::from("test-router-auth"), + ready: ready_site_controller_flag(), + }; + reconcile_local_framework_routes(&direct_controller_app.control) + .await + .expect("direct-local controller should publish delegated framework routes"); + let stale_err = match super::site_controller::execute_site_controller_framework_mutate( + &direct_controller_app, + &direct_record, + &direct_state, + super::ccs_api::FrameworkComponentMutateRequest::CreateChild(CreateChildRequest { + template: "root_worker".to_string(), + name: "sibling".to_string(), + manifest: None, + config: BTreeMap::new(), + bindings: BTreeMap::new(), + }), + ) + .await + { + Ok(_) => panic!("the delegated caller site should reject stale local delivery"), + Err(err) => err, + }; + assert_eq!(stale_err.0.code, ProtocolErrorCode::ControlStateUnavailable); + assert!( + stale_err + .0 + .message + .contains("router framework route overlays are stale"), + "unexpected stale-delivery error: {}", + stale_err.0.message + ); + + let mut compose_state = authoritative; + localize_framework_control_state(&mut compose_state, "compose_local") + .expect("compose-local state should localize"); + let compose_record = compose_state + .capability_instances + .values() + .find(|record| record.recipient_component_moniker == "/parent/delegate") + .cloned() + .expect("compose-local controller should retain the delegated capability instance"); + let compose_state_path = dir.path().join("compose-state.json"); + write_control_state(&compose_state_path, &compose_state).expect("compose state should write"); + let mut compose_plan = + test_control_state_app(&dir, compose_state.clone(), compose_state_path.clone()) + .controller_plan + .as_ref() + .clone(); + compose_plan.site_id = "compose_local".to_string(); + compose_plan.kind = SiteKind::Direct; + compose_plan.peer_site_router_urls = + BTreeMap::from([("direct_local".to_string(), peer_router_base_url)]); + compose_plan.peer_router_mesh_addrs = + BTreeMap::from([("direct_local".to_string(), "127.0.0.1:24000".to_string())]); + let compose_controller_app = SiteControllerApp { + control: ControlStateApp { + control_state: Arc::new(Mutex::new(compose_state.clone())), + controller_plan: Arc::new(compose_plan), + ..test_control_state_app(&dir, compose_state.clone(), compose_state_path) + }, + router_auth_token: Arc::::from("test-router-auth"), + ready: ready_site_controller_flag(), + }; + let response = super::site_controller::execute_site_controller_framework_mutate( + &compose_controller_app, + &compose_record, + &compose_state, + super::ccs_api::FrameworkComponentMutateRequest::CreateChild(CreateChildRequest { + template: "root_worker".to_string(), + name: "sibling".to_string(), + manifest: None, + config: BTreeMap::new(), + bindings: BTreeMap::new(), + }), + ) + .await + .expect("forwarded realm create should succeed on the authority site controller"); + let super::ccs_api::FrameworkComponentMutateResponse::CreateChild(response) = response else { + panic!("framework mutate response should be a create result"); + }; + assert_eq!( + response.child.selector, "children.sibling", + "the authority controller should create the child under the forwarded root realm", + ); + let persisted = compose_controller_app + .control + .control_state + .lock() + .await + .clone(); + assert!( + persisted + .live_children + .iter() + .any(|child| child.name == "sibling" + && child_runtime_site_id(child) + .as_deref() + .is_ok_and(|site_id| site_id == "compose_local")), + "the forwarded root child should be owned by the compose-local controller state" + ); +} + #[tokio::test] async fn capability_instance_auth_and_snapshot_scope_are_enforced() { let dir = TempDir::new().expect("temp dir"); @@ -5175,6 +6147,7 @@ fn shared_cross_site_link_is_retained_while_another_child_still_needs_it() { ), next_dynamic_capability_grant_id: 0, dynamic_capability_grants: BTreeMap::new(), + dynamic_capability_grant_authority_sites: BTreeMap::new(), dynamic_capability_journal: Vec::new(), capability_instances: BTreeMap::new(), journal: Vec::new(), @@ -7533,6 +8506,7 @@ volumes: "/site/compose-site/router", &BTreeMap::new(), &BTreeMap::new(), + &BTreeMap::new(), Some("unix:///amber/control/router-control.sock"), Some("127.0.0.1:24000"), &site_state_root.join("site-controller-state.json"), @@ -7587,6 +8561,14 @@ volumes: .get(serde_yaml::Value::String("volumes".to_string())) .and_then(serde_yaml::Value::as_sequence) .expect("controller service should mount volumes"); + let extra_hosts = service + .get(serde_yaml::Value::String("extra_hosts".to_string())) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller service should set extra_hosts"); + let healthcheck = service + .get(serde_yaml::Value::String("healthcheck".to_string())) + .and_then(serde_yaml::Value::as_mapping) + .expect("controller service should define a healthcheck"); assert!(volumes.iter().any(|value| { value.as_str() == Some(&format!( @@ -7605,6 +8587,22 @@ volumes: .iter() .any(|value| value.as_str() == Some("/var/run/docker.sock:/var/run/docker.sock")) ); + assert!( + extra_hosts + .iter() + .any(|value| value.as_str() == Some("host.docker.internal:host-gateway")), + "compose site controller should resolve host.docker.internal inside the site network", + ); + let healthcheck_test = healthcheck + .get(serde_yaml::Value::String("test".to_string())) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller healthcheck should define a test command"); + assert!( + healthcheck_test.iter().any(|value| value + .as_str() + .is_some_and(|value| value.contains("/healthz"))), + "compose site controller healthcheck should wait for the controller readiness endpoint", + ); } #[test] @@ -7654,6 +8652,7 @@ spec: "/site/kube-site/router", &BTreeMap::new(), &BTreeMap::new(), + &BTreeMap::new(), Some("amber-router:24100"), Some("127.0.0.1:24000"), &site_state_root.join("site-controller-state.json"), @@ -7759,6 +8758,7 @@ fn local_site_manager_state_uses_controller_plan_when_host_state_is_absent() { ), next_dynamic_capability_grant_id: 0, dynamic_capability_grants: BTreeMap::new(), + dynamic_capability_grant_authority_sites: BTreeMap::new(), dynamic_capability_journal: Vec::new(), live_children: Vec::new(), pending_creates: Vec::new(), From 5cda0c313fc504a29ce900c71a6711e7b495cab7 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Mon, 13 Apr 2026 16:00:21 +0800 Subject: [PATCH 04/25] Fix post-rebase integration drift --- cli/src/direct_runtime/runtime.rs | 1 + cli/src/mixed_run.rs | 26 +++++++++++++++++++ cli/src/run_logs.rs | 4 +-- .../src/default_runtime/dynamic_routes.rs | 1 + .../src/default_runtime/site_artifacts.rs | 1 + runtime/site-controller/src/tests.rs | 1 + 6 files changed, 32 insertions(+), 2 deletions(-) diff --git a/cli/src/direct_runtime/runtime.rs b/cli/src/direct_runtime/runtime.rs index 6ea1e18f..e4c0d8e9 100644 --- a/cli/src/direct_runtime/runtime.rs +++ b/cli/src/direct_runtime/runtime.rs @@ -1420,6 +1420,7 @@ mod tests { }], outbound: vec![OutboundRoute { route_id: "out".to_string(), + rewrite_route_id: None, slot: "compose".to_string(), capability_kind: None, capability_profile: None, diff --git a/cli/src/mixed_run.rs b/cli/src/mixed_run.rs index cd655fcc..97b1ebea 100644 --- a/cli/src/mixed_run.rs +++ b/cli/src/mixed_run.rs @@ -36,6 +36,7 @@ use amber_site_controller::{ use base64::Engine as _; use miette::{Context as _, IntoDiagnostic as _, Result}; use serde::{Deserialize, Serialize}; +use serde_json::Value as JsonValue; use tokio::time::{Instant, sleep}; use url::Url; @@ -246,6 +247,8 @@ struct LaunchBundleObservability { state_path: Option, #[serde(default, skip_serializing_if = "Option::is_none")] requests_log: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + events_ndjson: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] launch_commands: Vec, } @@ -321,6 +324,23 @@ pub(crate) struct ObservabilityReceipt { pub(crate) sink_pid: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) requests_log: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) events_ndjson: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct PersistedTraceEvent { + pub(crate) schema: String, + pub(crate) version: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) observed_at_unix_nano: Option, + pub(crate) message: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) severity: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) trace_id: Option, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub(crate) attributes: BTreeMap, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -542,6 +562,7 @@ struct ObservabilitySinkPlan { listen_addr: String, advertise_endpoint: String, requests_log: String, + events_ndjson: String, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -575,6 +596,10 @@ struct RunOutsideProxyContext { #[derive(Clone, Debug)] struct RunOutsideExport { site_id: String, + component: String, + provide: String, + capability_kind: Option, + capability_profile: Option, protocol: String, } @@ -1034,6 +1059,7 @@ pub(crate) async fn run_observability_sink(plan_path: PathBuf) -> Result<()> { endpoint: plan.advertise_endpoint.clone(), sink_pid: Some(std::process::id()), requests_log: Some(plan.requests_log.clone()), + events_ndjson: Some(plan.events_ndjson.clone()), }, )?; diff --git a/cli/src/run_logs.rs b/cli/src/run_logs.rs index eac9fd36..4bb297a3 100644 --- a/cli/src/run_logs.rs +++ b/cli/src/run_logs.rs @@ -636,7 +636,7 @@ impl NormalizedTraceInteraction { parse_story_actors(story).or_else(|| parse_story_receivers(story))?; Some(Self { severity: severity - .filter(|severity| !severity.eq_ignore_ascii_case("info")) + .filter(|severity: &&str| !severity.eq_ignore_ascii_case("info")) .map(ToString::to_string), actor, recipient, @@ -996,7 +996,7 @@ fn event_attr_str<'a>(event: &'a PersistedTraceEvent, key: &str) -> Option<&'a s fn event_attr_bool(event: &PersistedTraceEvent, key: &str) -> Option { match event.attributes.get(key)? { - JsonValue::Bool(value) => Some(*value), + JsonValue::Bool(value) => Some(value.to_owned()), _ => None, } } diff --git a/runtime/site-controller/src/default_runtime/dynamic_routes.rs b/runtime/site-controller/src/default_runtime/dynamic_routes.rs index efc7ca7f..c69800fe 100644 --- a/runtime/site-controller/src/default_runtime/dynamic_routes.rs +++ b/runtime/site-controller/src/default_runtime/dynamic_routes.rs @@ -1421,6 +1421,7 @@ mod direct_input_tests { inbound: Vec::new(), outbound: vec![OutboundRoute { route_id: "old".to_string(), + rewrite_route_id: None, slot: "api".to_string(), capability_kind: Some("http".to_string()), capability_profile: None, diff --git a/runtime/site-controller/src/default_runtime/site_artifacts.rs b/runtime/site-controller/src/default_runtime/site_artifacts.rs index 6936b6b4..0a3bcd66 100644 --- a/runtime/site-controller/src/default_runtime/site_artifacts.rs +++ b/runtime/site-controller/src/default_runtime/site_artifacts.rs @@ -1612,6 +1612,7 @@ fn inject_site_controller_peer_router_routes_into_plan( } router.config.outbound.push(OutboundRoute { route_id: site_controller_internal_route_id(&route.site_id), + rewrite_route_id: None, slot: route.site_id.clone(), capability_kind: None, capability_profile: None, diff --git a/runtime/site-controller/src/tests.rs b/runtime/site-controller/src/tests.rs index 19177e97..9e56f5aa 100644 --- a/runtime/site-controller/src/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -3921,6 +3921,7 @@ async fn install_dynamic_caps_origin_fixture(app: &ControlStateApp) -> tokio::ta Vec::new(), vec![OutboundRoute { route_id: "provider-route".to_string(), + rewrite_route_id: None, slot: "upstream".to_string(), capability_kind: Some("http".to_string()), capability_profile: None, From cdde0032627a5bf88b302e40e9086e8bf827cb58 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Mon, 13 Apr 2026 23:13:06 +0800 Subject: [PATCH 05/25] Fix some more tests --- cli/src/direct_runtime/runtime.rs | 25 +- cli/src/main.rs | 2 +- cli/src/mixed_run.rs | 16 +- cli/src/mixed_run/launch_bundle.rs | 54 +- cli/src/mixed_run/supervisor.rs | 28 +- cli/src/mixed_run/tests.rs | 86 ++- cli/src/vm_runtime.rs | 2 +- cli/src/vm_runtime/state.rs | 25 +- cli/tests/mixed_run/support.rs | 36 +- compiler/src/targets/mesh/kubernetes/mod.rs | 17 +- .../mesh/kubernetes/tests/rendering.rs | 10 + docker/amber-site-controller/Dockerfile | 12 +- .../site-controller/src/default_runtime.rs | 11 +- .../src/default_runtime/child_runtime.rs | 17 +- .../src/default_runtime/dynamic_routes.rs | 12 +- .../default_runtime/kubernetes_controller.rs | 177 ++++-- .../src/default_runtime/site_artifacts.rs | 10 - .../default_runtime/site_runtime_support.rs | 118 +++- runtime/site-controller/src/lib.rs | 12 +- runtime/site-controller/src/orchestration.rs | 379 +++++++++---- runtime/site-controller/src/planner.rs | 44 +- .../site-controller/src/site_controller.rs | 81 ++- runtime/site-controller/src/tests.rs | 536 +++++++++++++++++- 23 files changed, 1417 insertions(+), 293 deletions(-) diff --git a/cli/src/direct_runtime/runtime.rs b/cli/src/direct_runtime/runtime.rs index e4c0d8e9..f0c14c37 100644 --- a/cli/src/direct_runtime/runtime.rs +++ b/cli/src/direct_runtime/runtime.rs @@ -1025,7 +1025,7 @@ pub(crate) fn allocate_direct_runtime_port( )); } for _ in 0..256 { - let port = pick_free_port()?; + let port = amber_site_controller::reserve_loopback_port()?; if reserved.insert(port) { return Ok(port); } @@ -1035,12 +1035,6 @@ pub(crate) fn allocate_direct_runtime_port( )) } -pub(crate) fn pick_free_port() -> Result { - let listener = TcpListener::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0)) - .into_diagnostic()?; - Ok(listener.local_addr().into_diagnostic()?.port()) -} - pub(crate) fn rewrite_direct_mesh_peer_addrs( config: &mut MeshConfigPublic, mesh_port_by_peer_id: &HashMap, @@ -1459,4 +1453,21 @@ mod tests { "direct sidecar control socket path must fit within unix socket limits: {rendered}", ); } + + #[test] + fn allocate_direct_runtime_port_avoids_shared_loopback_reservations() { + let reserved_port = + amber_site_controller::reserve_loopback_port().expect("shared loopback port"); + let mut reserved = BTreeSet::new(); + + for _ in 0..32 { + let port = allocate_direct_runtime_port(&mut reserved, None) + .expect("direct runtime port allocation should succeed"); + assert_ne!( + port, reserved_port, + "direct runtime should not reuse loopback ports already reserved by mixed-run \ + infrastructure", + ); + } + } } diff --git a/cli/src/main.rs b/cli/src/main.rs index c948de8a..32aacc49 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -20,7 +20,7 @@ use std::{ future::Future, io, io::{IsTerminal as _, Write as _}, - net::{IpAddr, Ipv4Addr, Shutdown, SocketAddr, TcpListener, TcpStream}, + net::{IpAddr, Ipv4Addr, Shutdown, SocketAddr, TcpStream}, path::{Path, PathBuf}, process::{Child, Stdio}, }; diff --git a/cli/src/mixed_run.rs b/cli/src/mixed_run.rs index 97b1ebea..784ee0fb 100644 --- a/cli/src/mixed_run.rs +++ b/cli/src/mixed_run.rs @@ -75,7 +75,6 @@ const OUTSIDE_PROXY_PLAN_SCHEMA: &str = "amber.run.outside_proxy"; const OUTSIDE_PROXY_PLAN_VERSION: u32 = 1; const OUTSIDE_PROXY_STATE_SCHEMA: &str = "amber.run.outside_proxy_state"; const OUTSIDE_PROXY_STATE_VERSION: u32 = 1; - const ROUTER_CONTROL_TIMEOUT: Duration = Duration::from_secs(30); const SUPERVISOR_POLL_INTERVAL: Duration = Duration::from_millis(500); const RESTART_BACKOFF: Duration = Duration::from_secs(1); @@ -452,8 +451,6 @@ pub(crate) struct SiteSupervisorPlan { site_controller_plan_path: Option, #[serde(default, skip_serializing_if = "Option::is_none")] site_controller_url: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - controller_route_ports: Vec, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] launch_env: BTreeMap, } @@ -1083,10 +1080,7 @@ pub(crate) async fn run_observability_sink(plan_path: PathBuf) -> Result<()> { } pub(crate) fn reserve_loopback_port() -> Result { - let listener = TcpListener::bind(("127.0.0.1", 0)) - .into_diagnostic() - .wrap_err("failed to allocate a loopback port")?; - Ok(listener.local_addr().into_diagnostic()?.port()) + amber_site_controller::reserve_loopback_port() } pub(crate) fn new_run_id() -> String { @@ -1125,6 +1119,14 @@ pub(crate) fn site_controller_plan_path(site_state_root: &Path) -> PathBuf { site_state_root.join("site-controller-plan.json") } +pub(crate) fn site_existing_peer_ports_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("existing-peer-ports.json") +} + +pub(crate) fn site_existing_peer_identities_path(site_state_root: &Path) -> PathBuf { + site_state_root.join("existing-peer-identities.json") +} + pub(crate) fn desired_links_path(site_state_root: &Path) -> PathBuf { site_state_root.join("desired-links.json") } diff --git a/cli/src/mixed_run/launch_bundle.rs b/cli/src/mixed_run/launch_bundle.rs index b86cc112..1dd62e2c 100644 --- a/cli/src/mixed_run/launch_bundle.rs +++ b/cli/src/mixed_run/launch_bundle.rs @@ -241,18 +241,18 @@ pub(super) fn materialize_launch_bundle( let mut peer_site_router_urls = BTreeMap::new(); let mut peer_router_identities = BTreeMap::new(); let mut peer_router_mesh_addrs = BTreeMap::new(); - let mut controller_route_ports = Vec::new(); let mut controller_routes = Vec::new(); - let peer_router_host = host_service_host_for_consumer(site.site_plan.site.kind); for (peer_site_id, peer_site_plan) in &run_plan.sites { if peer_site_id == site_id { continue; } let route_port = reserve_loopback_port()?; - controller_route_ports.push(route_port); peer_site_router_urls.insert( peer_site_id.clone(), - format!("http://{peer_router_host}:{route_port}"), + amber_site_controller::site_controller_peer_router_url( + site.site_plan.site.kind, + route_port, + ), ); peer_router_identities.insert( peer_site_id.clone(), @@ -284,19 +284,21 @@ pub(super) fn materialize_launch_bundle( }); } + write_json( + &site_existing_peer_ports_path(&site.site_state_root), + &peer_router_ports_by_identity(&peer_router_identities, &peer_router_mesh_addrs)?, + )?; + write_json( + &site_existing_peer_identities_path(&site.site_state_root), + &peer_router_identities_by_id(&peer_router_identities), + )?; + amber_site_controller::inject_site_controller_peer_router_routes( &site.artifact_dir, site_id, &allowed_issuers, &controller_routes, )?; - if site.site_plan.site.kind == SiteKind::Compose { - amber_site_controller::add_compose_router_published_route_ports( - &site.artifact_dir, - &controller_route_ports, - )?; - } - site.base_supervisor_plan.controller_route_ports = controller_route_ports; let local_router_control = site_controller_local_router_control(site.site_plan.site.kind, &site.artifact_dir); @@ -407,6 +409,36 @@ fn planned_router_identity( )) } +fn peer_router_ports_by_identity( + peer_router_identities: &BTreeMap, + peer_router_mesh_addrs: &BTreeMap, +) -> Result> { + peer_router_identities + .iter() + .map(|(site_id, identity)| { + let addr = peer_router_mesh_addrs.get(site_id).ok_or_else(|| { + miette::miette!("missing planned router mesh address for peer site `{site_id}`") + })?; + let port = addr + .parse::() + .into_diagnostic() + .wrap_err_with(|| format!("invalid planned router mesh address `{addr}`"))? + .port(); + Ok((identity.id.clone(), port)) + }) + .collect() +} + +fn peer_router_identities_by_id( + peer_router_identities: &BTreeMap, +) -> BTreeMap { + peer_router_identities + .values() + .cloned() + .map(|identity| (identity.id.clone(), identity)) + .collect() +} + pub(super) fn build_launch_bundle_manifest( run_id: &str, source_plan_path: Option<&Path>, diff --git a/cli/src/mixed_run/supervisor.rs b/cli/src/mixed_run/supervisor.rs index 4c050f3f..a788bfb4 100644 --- a/cli/src/mixed_run/supervisor.rs +++ b/cli/src/mixed_run/supervisor.rs @@ -263,7 +263,6 @@ pub(super) fn build_supervisor_plan( .site_controller_plan_path .map(|path| path.display().to_string()), site_controller_url: input.site_controller_url.map(ToOwned::to_owned), - controller_route_ports: Vec::new(), launch_env, }) } @@ -665,6 +664,10 @@ pub(super) async fn ensure_site_running( if runtime.site_process.is_none() { runtime.last_start_attempt = Some(Instant::now()); runtime.ready_since = None; + let existing_peer_ports_path = + site_existing_peer_ports_path(Path::new(&plan.site_state_root)); + let existing_peer_identities_path = + site_existing_peer_identities_path(Path::new(&plan.site_state_root)); runtime.site_process = Some(spawn_runtime_process( &PathBuf::from(&plan.site_state_root), "site.log", @@ -684,6 +687,14 @@ pub(super) async fn ensure_site_running( if let Some(port) = plan.router_mesh_port { cmd.arg("--router-mesh-port").arg(port.to_string()); } + if existing_peer_ports_path.is_file() { + cmd.arg("--existing-peer-ports") + .arg(&existing_peer_ports_path); + } + if existing_peer_identities_path.is_file() { + cmd.arg("--existing-peer-identities") + .arg(&existing_peer_identities_path); + } }, )?); } @@ -692,6 +703,10 @@ pub(super) async fn ensure_site_running( if runtime.site_process.is_none() { runtime.last_start_attempt = Some(Instant::now()); runtime.ready_since = None; + let existing_peer_ports_path = + site_existing_peer_ports_path(Path::new(&plan.site_state_root)); + let existing_peer_identities_path = + site_existing_peer_identities_path(Path::new(&plan.site_state_root)); runtime.site_process = Some(spawn_runtime_process( &PathBuf::from(&plan.site_state_root), "site.log", @@ -711,6 +726,14 @@ pub(super) async fn ensure_site_running( if let Some(port) = plan.router_mesh_port { cmd.arg("--router-mesh-port").arg(port.to_string()); } + if existing_peer_ports_path.is_file() { + cmd.arg("--existing-peer-ports") + .arg(&existing_peer_ports_path); + } + if existing_peer_identities_path.is_file() { + cmd.arg("--existing-peer-identities") + .arg(&existing_peer_identities_path); + } }, )?); } @@ -1779,9 +1802,6 @@ pub(super) fn spawn_port_forward(plan: &SiteSupervisorPlan) -> Result { .arg("deploy/amber-router") .arg(format!("{mesh_port}:24000")) .arg(format!("{control_port}:24100")); - for port in &plan.controller_route_ports { - cmd.arg(format!("{port}:{port}")); - } cmd.stdout(Stdio::from(log)).stderr(Stdio::from(log_err)); cmd.spawn() .into_diagnostic() diff --git a/cli/src/mixed_run/tests.rs b/cli/src/mixed_run/tests.rs index cdc50696..fa45191e 100644 --- a/cli/src/mixed_run/tests.rs +++ b/cli/src/mixed_run/tests.rs @@ -24,6 +24,14 @@ fn site_state_paths_are_site_scoped() { site_controller_plan_path(Path::new("/tmp/amber-run/state/site-a")), Path::new("/tmp/amber-run/state/site-a/site-controller-plan.json") ); + assert_eq!( + site_existing_peer_ports_path(Path::new("/tmp/amber-run/state/site-a")), + Path::new("/tmp/amber-run/state/site-a/existing-peer-ports.json") + ); + assert_eq!( + site_existing_peer_identities_path(Path::new("/tmp/amber-run/state/site-a")), + Path::new("/tmp/amber-run/state/site-a/existing-peer-identities.json") + ); } #[test] @@ -68,6 +76,55 @@ fn site_controller_local_router_control_uses_backend_local_control_targets() { ); } +#[test] +fn site_controller_peer_router_urls_are_local_to_the_controller_site() { + assert_eq!( + amber_site_controller::site_controller_peer_router_url(SiteKind::Direct, 37046), + "http://127.0.0.1:37046" + ); + assert_eq!( + amber_site_controller::site_controller_peer_router_url(SiteKind::Vm, 37046), + "http://127.0.0.1:37046" + ); + assert_eq!( + amber_site_controller::site_controller_peer_router_url(SiteKind::Compose, 37046), + "http://amber-router:37046" + ); + assert_eq!( + amber_site_controller::site_controller_peer_router_url(SiteKind::Kubernetes, 37046), + "http://amber-router:37046" + ); +} + +#[test] +fn site_controller_image_includes_the_amber_cli_binary() { + let dockerfile = fs::read_to_string( + Path::new(env!("CARGO_MANIFEST_DIR")).join("../docker/amber-site-controller/Dockerfile"), + ) + .expect("site-controller Dockerfile should read"); + assert!( + dockerfile.contains("cargo build --locked --release -p amber-cli -p amber-site-controller") + || dockerfile.contains("cargo build --locked -p amber-cli -p amber-site-controller"), + "site-controller image must build the amber CLI so containerized controllers can spawn \ + amber subcommands:\n{dockerfile}" + ); + assert!( + dockerfile.contains("COPY --from=builder /out/amber /usr/local/bin/amber"), + "site-controller image must ship the amber CLI binary alongside \ + amber-site-controller:\n{dockerfile}" + ); + assert!( + dockerfile.contains("COPY examples ./examples"), + "site-controller image must include the examples tree so amber-cli can satisfy \ + cli/build.rs:\n{dockerfile}" + ); + assert!( + dockerfile.contains("COPY README.md ./"), + "site-controller image must include the workspace README because amber-cli embeds \ + it:\n{dockerfile}" + ); +} + #[test] fn site_controller_command_prefers_fresh_amber_binary_under_cargo_tests() { let temp = TempDir::new().expect("temp dir"); @@ -117,7 +174,6 @@ fn local_site_controller_addr_requires_loopback_http() { observability_endpoint: None, site_controller_plan_path: None, site_controller_url: Some("http://127.0.0.1:24200".to_string()), - controller_route_ports: Vec::new(), launch_env: BTreeMap::new(), }; @@ -169,7 +225,6 @@ fn local_site_controller_ready_waits_for_http_listener() { observability_endpoint: None, site_controller_plan_path: None, site_controller_url: Some(format!("http://127.0.0.1:{}", addr.port())), - controller_route_ports: Vec::new(), launch_env: BTreeMap::new(), }; @@ -218,7 +273,6 @@ fn compose_site_controller_container_name_uses_compose_project() { observability_endpoint: None, site_controller_plan_path: None, site_controller_url: Some("http://amber-site-controller:4100".to_string()), - controller_route_ports: vec![60066], launch_env: BTreeMap::new(), }; @@ -227,3 +281,29 @@ fn compose_site_controller_container_name_uses_compose_project() { Some("amber_run_compose-site-amber-site-controller-1") ); } + +#[test] +fn reserve_loopback_port_keeps_allocations_unique_within_one_process() { + let mut ports = BTreeSet::new(); + for _ in 0..64 { + let port = reserve_loopback_port().expect("loopback port reservation should succeed"); + assert!( + ports.insert(port), + "loopback port allocator reused {port} within one process" + ); + } +} + +#[test] +fn reserve_loopback_port_shares_allocator_with_site_controller_runtime() { + let mixed_run_port = + reserve_loopback_port().expect("mixed-run loopback port reservation should succeed"); + for _ in 0..32 { + let runtime_port = amber_site_controller::reserve_loopback_port() + .expect("site-controller loopback port reservation should succeed"); + assert_ne!( + runtime_port, mixed_run_port, + "mixed-run and site-controller loopback reservations must use the same shared pool", + ); + } +} diff --git a/cli/src/vm_runtime.rs b/cli/src/vm_runtime.rs index fd7d7bd5..8ab62cd2 100644 --- a/cli/src/vm_runtime.rs +++ b/cli/src/vm_runtime.rs @@ -5,7 +5,7 @@ use std::{ fs::{self, File}, hash::{Hash as _, Hasher as _}, io::{Seek as _, SeekFrom, Write as _}, - net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener}, + net::SocketAddr, path::{Path, PathBuf}, process::{Command as ProcessCommand, ExitStatus, Stdio}, }; diff --git a/cli/src/vm_runtime/state.rs b/cli/src/vm_runtime/state.rs index 47b92072..90d04d42 100644 --- a/cli/src/vm_runtime/state.rs +++ b/cli/src/vm_runtime/state.rs @@ -379,7 +379,7 @@ pub(crate) fn allocate_runtime_port( )); } for _ in 0..256 { - let port = pick_free_port()?; + let port = amber_site_controller::reserve_loopback_port()?; if reserved.insert(port) { return Ok(port); } @@ -389,12 +389,6 @@ pub(crate) fn allocate_runtime_port( )) } -pub(crate) fn pick_free_port() -> Result { - let listener = TcpListener::bind(SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0)) - .into_diagnostic()?; - Ok(listener.local_addr().into_diagnostic()?.port()) -} - pub(crate) fn rewrite_mesh_peer_addrs( config: &mut MeshConfigPublic, mesh_port_by_peer_id: &HashMap, @@ -1055,4 +1049,21 @@ mod tests { "vm sidecar control socket path must fit within unix socket limits: {rendered}", ); } + + #[test] + fn allocate_runtime_port_avoids_shared_loopback_reservations() { + let reserved_port = + amber_site_controller::reserve_loopback_port().expect("shared loopback port"); + let mut reserved = BTreeSet::new(); + + for _ in 0..32 { + let port = allocate_runtime_port(&mut reserved, None) + .expect("vm runtime port allocation should succeed"); + assert_ne!( + port, reserved_port, + "vm runtime should not reuse loopback ports already reserved by mixed-run \ + infrastructure", + ); + } + } } diff --git a/cli/tests/mixed_run/support.rs b/cli/tests/mixed_run/support.rs index 5a014193..40973eab 100644 --- a/cli/tests/mixed_run/support.rs +++ b/cli/tests/mixed_run/support.rs @@ -201,8 +201,40 @@ pub(crate) fn temp_output_dir(prefix: &str) -> TestTempDir { } pub(crate) fn pick_free_port() -> u16 { - let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).unwrap(); - listener.local_addr().unwrap().port() + const TEST_PORT_RANGE_START: u16 = 20000; + const TEST_PORT_RANGE_END: u16 = 30000; + static RESERVED_TEST_PORTS: OnceLock>> = OnceLock::new(); + + let reserved = RESERVED_TEST_PORTS.get_or_init(|| Mutex::new(BTreeSet::new())); + let mut reserved = reserved + .lock() + .expect("test port allocator should not be poisoned"); + let span = u32::from(TEST_PORT_RANGE_END - TEST_PORT_RANGE_START); + let mut next = + TEST_PORT_RANGE_START + (std::process::id() % span) as u16 + reserved.len() as u16; + for _ in 0..usize::from(TEST_PORT_RANGE_END - TEST_PORT_RANGE_START) { + if next >= TEST_PORT_RANGE_END { + next = TEST_PORT_RANGE_START; + } + let port = next; + next += 1; + if reserved.contains(&port) { + continue; + } + match TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], port))) { + Ok(listener) => { + drop(listener); + reserved.insert(port); + return port; + } + Err(_) => continue, + } + } + panic!( + "failed to allocate a unique mixed-run test port in {}-{}", + TEST_PORT_RANGE_START, + TEST_PORT_RANGE_END - 1 + ); } pub(crate) fn docker_host_ip() -> String { diff --git a/compiler/src/targets/mesh/kubernetes/mod.rs b/compiler/src/targets/mesh/kubernetes/mod.rs index 5d5a79e9..e607e7b9 100644 --- a/compiler/src/targets/mesh/kubernetes/mod.rs +++ b/compiler/src/targets/mesh/kubernetes/mod.rs @@ -314,16 +314,7 @@ pub(crate) fn emit_kubernetes_artifact_with_options( name: mesh_secret_name(ROUTER_NAME), namespace: None, }, - |router_config| { - if let Some(control_listen) = router_config.control_listen { - router_config.control_listen = Some( - format!("127.0.0.1:{}", control_listen.port()) - .parse() - .expect("control listen"), - ); - } - router_config.control_allow = Some(vec!["127.0.0.1".to_string(), "::1".to_string()]); - }, + |_| {}, ) .map_err(|err| ReporterError::new(err.to_string()))?; @@ -1155,6 +1146,12 @@ pub(crate) fn emit_kubernetes_artifact_with_options( ); if !router_service_ports.is_empty() { + router_service_ports.push(ServicePort { + name: "control".to_string(), + port: ROUTER_CONTROL_PORT_BASE, + target_port: ROUTER_CONTROL_PORT_BASE, + protocol: "TCP", + }); let service = Service::new( ROUTER_NAME, &namespace, diff --git a/compiler/src/targets/mesh/kubernetes/tests/rendering.rs b/compiler/src/targets/mesh/kubernetes/tests/rendering.rs index 1b32aa0a..d6bd9310 100644 --- a/compiler/src/targets/mesh/kubernetes/tests/rendering.rs +++ b/compiler/src/targets/mesh/kubernetes/tests/rendering.rs @@ -163,6 +163,16 @@ fn kubernetes_emits_router_for_external_slots() { .get(&PathBuf::from("04-services/amber-router.yaml")) .expect("router service"); assert!(router_service.contains("port: 24000"), "{router_service}"); + assert!(router_service.contains("port: 24100"), "{router_service}"); + assert!(router_service.contains("name: control"), "{router_service}"); + let mesh_provision = artifact + .files + .get(&PathBuf::from("01-configmaps/amber-mesh-provision.yaml")) + .expect("mesh provision configmap"); + assert!( + mesh_provision.contains("\"control_listen\": \"0.0.0.0:24100\""), + "{mesh_provision}" + ); let router_env = artifact .files diff --git a/docker/amber-site-controller/Dockerfile b/docker/amber-site-controller/Dockerfile index eb593480..33169dc2 100644 --- a/docker/amber-site-controller/Dockerfile +++ b/docker/amber-site-controller/Dockerfile @@ -27,6 +27,7 @@ RUN case "${TARGETARCH}" in \ FROM base-builder AS builder COPY Cargo.toml Cargo.lock ./ +COPY README.md ./ COPY cli/Cargo.toml cli/ COPY compiler/config/Cargo.toml compiler/config/ COPY compiler/Cargo.toml compiler/ @@ -53,6 +54,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ COPY cli ./cli COPY compiler ./compiler +COPY examples ./examples COPY images ./images COPY runtime/helper ./runtime/helper COPY runtime/mesh ./runtime/mesh @@ -67,18 +69,20 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/app/target,sharing=locked \ target=$(cat /tmp/rust-target) && \ if [ "$BUILD_MODE" = "release" ]; then \ - cargo build --locked --release -p amber-site-controller --target "${target}"; \ + cargo build --locked --release -p amber-cli -p amber-site-controller --target "${target}"; \ build_dir=release; \ else \ - cargo build --locked -p amber-site-controller --target "${target}"; \ + cargo build --locked -p amber-cli -p amber-site-controller --target "${target}"; \ build_dir=debug; \ fi && \ + install -D -m 0755 /app/target/"${target}"/"${build_dir}"/amber /out/amber && \ install -D -m 0755 /app/target/"${target}"/"${build_dir}"/amber-site-controller /out/amber-site-controller FROM docker:28-cli AS runtime RUN apk add --no-cache kubectl -COPY --from=builder /out/amber-site-controller /amber-site-controller +COPY --from=builder /out/amber /usr/local/bin/amber +COPY --from=builder /out/amber-site-controller /usr/local/bin/amber-site-controller -ENTRYPOINT ["/amber-site-controller"] +ENTRYPOINT ["/usr/local/bin/amber-site-controller"] diff --git a/runtime/site-controller/src/default_runtime.rs b/runtime/site-controller/src/default_runtime.rs index 2bd5519f..e204911b 100644 --- a/runtime/site-controller/src/default_runtime.rs +++ b/runtime/site-controller/src/default_runtime.rs @@ -56,13 +56,13 @@ pub use self::{ compose_controller::inject_compose_site_controller, kubernetes_controller::inject_kubernetes_site_controller, site_artifacts::{ - SiteControllerPeerRouterRoute, add_compose_router_published_route_ports, - inject_site_controller_peer_router_routes, set_compose_router_published_mesh_port, - set_site_artifact_mesh_identity_seed, + SiteControllerPeerRouterRoute, inject_site_controller_peer_router_routes, + set_compose_router_published_mesh_port, set_site_artifact_mesh_identity_seed, }, site_runtime_support::{ host_service_bind_addr_for_consumer, observability_endpoint_for_site, - prepare_kubernetes_artifact_namespace, router_mesh_addr_for_consumer, walk_files, + prepare_kubernetes_artifact_namespace, reserve_loopback_port, + router_mesh_addr_for_consumer, site_controller_peer_router_url, walk_files, }, }; use self::{ @@ -81,7 +81,6 @@ use crate::{ http::{read_json, write_json}, planner::{ LocalChildRuntimeSpec, build_desired_site_artifact_files, build_local_child_runtime_spec, - child_runtime_site_id, }, runtime_api::{SharedSiteControllerRuntime, SiteControllerRuntime}, state::{FrameworkControlState, LiveChildRecord}, @@ -198,8 +197,6 @@ struct SiteSupervisorPlan { site_controller_plan_path: Option, #[serde(default, skip_serializing_if = "Option::is_none")] site_controller_url: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - controller_route_ports: Vec, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] launch_env: BTreeMap, } diff --git a/runtime/site-controller/src/default_runtime/child_runtime.rs b/runtime/site-controller/src/default_runtime/child_runtime.rs index 1caa07df..b12c0a5b 100644 --- a/runtime/site-controller/src/default_runtime/child_runtime.rs +++ b/runtime/site-controller/src/default_runtime/child_runtime.rs @@ -281,15 +281,7 @@ fn local_child_runtime_spec( state: &FrameworkControlState, child: &LiveChildRecord, ) -> Result { - let site_id = child_runtime_site_id(child).map_err(|err| miette::miette!(err.message))?; - if site_id != plan.site_id { - return Err(miette::miette!( - "dynamic child `{}` targeted site `{site_id}` but runtime plan belongs to site `{}`", - child.name, - plan.site_id - )); - } - build_local_child_runtime_spec(state, child, &site_id) + build_local_child_runtime_spec(state, child, &plan.site_id) .map_err(|err| miette::miette!(err.message)) } @@ -1017,12 +1009,15 @@ pub(super) async fn site_controller_runtime_destroy_child( state: &FrameworkControlState, child: &LiveChildRecord, ) -> Result<()> { - let site_id = child_runtime_site_id(child).map_err(|err| miette::miette!(err.message))?; let child_id = child.child_id; let child = { let state = app.state.lock().await; state.children.get(&child_id).cloned() }; + if child.is_none() { + remove_dir_if_exists(&site_controller_runtime_child_root(&app.plan, child_id))?; + return Ok(()); + } if let Some(child) = child.as_ref() && matches!( app.plan.kind, @@ -1118,7 +1113,7 @@ pub(super) async fn site_controller_runtime_destroy_child( .await?; } SiteKind::Direct | SiteKind::Vm => { - let site_artifact_files = build_desired_site_artifact_files(state, &site_id) + let site_artifact_files = build_desired_site_artifact_files(state, &app.plan.site_id) .map_err(|err| miette::miette!(err.message))?; reconcile_site_proxy_metadata(Path::new(&app.plan.artifact_dir), &site_artifact_files)?; } diff --git a/runtime/site-controller/src/default_runtime/dynamic_routes.rs b/runtime/site-controller/src/default_runtime/dynamic_routes.rs index c69800fe..6e381926 100644 --- a/runtime/site-controller/src/default_runtime/dynamic_routes.rs +++ b/runtime/site-controller/src/default_runtime/dynamic_routes.rs @@ -1678,7 +1678,7 @@ mod direct_input_tests { } #[test] - fn compose_router_port_helpers_rewrite_mesh_publish_and_add_controller_ports() { + fn compose_router_port_helpers_rewrite_mesh_publish() { let temp = tempfile::tempdir().expect("tempdir"); fs::write( temp.path().join("compose.yaml"), @@ -1693,8 +1693,6 @@ services: .expect("compose yaml should write"); set_compose_router_published_mesh_port(temp.path(), 34000).expect("rewrite mesh port"); - add_compose_router_published_route_ports(temp.path(), &[34001, 34002]) - .expect("append controller ports"); let rendered = fs::read_to_string(temp.path().join("compose.yaml")).expect("compose yaml should read"); @@ -1702,13 +1700,5 @@ services: rendered.contains("0.0.0.0:34000:24000"), "mesh port should be rewritten to a deterministic host publish:\n{rendered}" ); - assert!( - rendered.contains("127.0.0.1:34001:34001"), - "controller route port 34001 should be published on loopback:\n{rendered}" - ); - assert!( - rendered.contains("127.0.0.1:34002:34002"), - "controller route port 34002 should be published on loopback:\n{rendered}" - ); } } diff --git a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs index 90150e0a..7cac53cd 100644 --- a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs +++ b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs @@ -1,5 +1,5 @@ use std::{ - collections::BTreeMap, + collections::{BTreeMap, BTreeSet}, fs, io::Write as _, net::SocketAddr, @@ -15,6 +15,7 @@ use super::*; const KUBERNETES_ROUTER_NAME: &str = "amber-router"; const KUBERNETES_ROUTER_MESH_PORT: u16 = 24000; const KUBERNETES_ROUTER_CONTROL_PORT: u16 = 24100; +const KUBERNETES_ROUTER_SERVICE_PATH: &str = "04-services/amber-router.yaml"; const KUBERNETES_ROUTER_NETPOL_PATH: &str = "05-networkpolicies/amber-router-netpol.yaml"; const KUBERNETES_CONTROLLER_SEED_CONFIGMAP_PATH: &str = "01-configmaps/amber-site-controller-seed.yaml"; @@ -179,7 +180,7 @@ pub fn inject_kubernetes_site_controller( "containers": [{ "name": SITE_CONTROLLER_SERVICE_NAME, "image": controller_image, - "command": ["--plan", KUBERNETES_CONTROLLER_PLAN_PATH], + "args": ["--plan", KUBERNETES_CONTROLLER_PLAN_PATH], "ports": [{ "name": "http", "containerPort": SITE_CONTROLLER_PORT, @@ -239,7 +240,10 @@ pub fn inject_kubernetes_site_controller( KUBERNETES_CONTROLLER_SERVICE_PATH, ], )?; - ensure_kubernetes_router_allows_site_controller_control(artifact_root) + ensure_kubernetes_router_allows_site_controller_ingress( + artifact_root, + &peer_router_route_ports(&embedded_plan)?, + ) } fn build_embedded_kubernetes_controller_plan(plan: &SiteControllerPlan) -> SiteControllerPlan { @@ -342,7 +346,81 @@ fn add_kubernetes_resource_paths(artifact_root: &Path, paths: &[&str]) -> Result .wrap_err_with(|| format!("failed to write {}", kustomization_path.display())) } -fn ensure_kubernetes_router_allows_site_controller_control(artifact_root: &Path) -> Result<()> { +fn peer_router_route_ports(plan: &SiteControllerPlan) -> Result> { + plan.peer_site_router_urls + .values() + .map(|url| { + let parsed = Url::parse(url) + .into_diagnostic() + .wrap_err_with(|| format!("invalid peer router url `{url}`"))?; + parsed + .port() + .ok_or_else(|| miette::miette!("peer router url `{url}` is missing a route port")) + }) + .collect() +} + +fn ensure_kubernetes_router_service_ports( + artifact_root: &Path, + route_ports: &BTreeSet, +) -> Result<()> { + if route_ports.is_empty() { + return Ok(()); + } + let path = artifact_root.join(KUBERNETES_ROUTER_SERVICE_PATH); + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid {}", path.display()))?; + let ports = document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("spec"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .and_then(|spec| spec.get_mut(yaml_string("ports"))) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!("router service {} is missing spec.ports", path.display()) + })?; + let existing = ports + .iter() + .filter_map(|entry| { + entry + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("port"))) + .and_then(serde_yaml::Value::as_u64) + .and_then(|port| u16::try_from(port).ok()) + }) + .collect::>(); + for port in route_ports { + if existing.contains(port) { + continue; + } + ports.push( + serde_yaml::to_value(json!({ + "name": format!("controller-route-{port}"), + "port": port, + "targetPort": port, + "protocol": "TCP", + })) + .into_diagnostic() + .wrap_err("failed to serialize router service route port")?, + ); + } + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(&path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + +fn ensure_kubernetes_router_allows_site_controller_ingress( + artifact_root: &Path, + route_ports: &BTreeSet, +) -> Result<()> { + ensure_kubernetes_router_service_ports(artifact_root, route_ports)?; let path = artifact_root.join(KUBERNETES_ROUTER_NETPOL_PATH); let raw = fs::read_to_string(&path) .into_diagnostic() @@ -363,43 +441,62 @@ fn ensure_kubernetes_router_allows_site_controller_control(artifact_root: &Path) ) })?; let controller_selector = kubernetes_controller_selector(); - let exists = ingress.iter().any(|rule| { + let controller_component = controller_selector + .get("amber.io/component") + .expect("selector must contain component"); + let mut required_ports = BTreeSet::from([KUBERNETES_ROUTER_CONTROL_PORT]); + required_ports.extend(route_ports.iter().copied()); + if let Some(rule) = ingress.iter_mut().find(|rule| { rule.as_mapping() - .and_then(|mapping| mapping.get(yaml_string("ports"))) + .and_then(|mapping| mapping.get(yaml_string("from"))) .and_then(serde_yaml::Value::as_sequence) - .is_some_and(|ports| { - ports.iter().any(|port| { - port.as_mapping() - .and_then(|entry| entry.get(yaml_string("port"))) - .and_then(serde_yaml::Value::as_i64) - == Some(i64::from(KUBERNETES_ROUTER_CONTROL_PORT)) + .is_some_and(|from| { + from.iter().any(|peer| { + peer.as_mapping() + .and_then(|mapping| mapping.get(yaml_string("podSelector"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|selector_value| selector_value.get(yaml_string("matchLabels"))) + .and_then(serde_yaml::Value::as_mapping) + .is_some_and(|labels| { + labels.get(yaml_string("amber.io/component")) + == Some(&yaml_string(controller_component)) + }) }) }) - && rule - .as_mapping() - .and_then(|mapping| mapping.get(yaml_string("from"))) - .and_then(serde_yaml::Value::as_sequence) - .is_some_and(|from| { - from.iter().any(|peer| { - peer.as_mapping() - .and_then(|mapping| mapping.get(yaml_string("podSelector"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|selector_value| { - selector_value.get(yaml_string("matchLabels")) - }) - .and_then(serde_yaml::Value::as_mapping) - .is_some_and(|labels| { - labels.get(yaml_string("amber.io/component")) - == Some(&yaml_string( - controller_selector - .get("amber.io/component") - .expect("selector must contain component"), - )) - }) - }) - }) - }); - if !exists { + }) { + let ports = rule + .as_mapping_mut() + .and_then(|mapping| mapping.get_mut(yaml_string("ports"))) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!( + "router network policy {} controller ingress rule is missing ports", + path.display() + ) + })?; + let existing_ports = ports + .iter() + .filter_map(|port| { + port.as_mapping() + .and_then(|entry| entry.get(yaml_string("port"))) + .and_then(serde_yaml::Value::as_u64) + .and_then(|port| u16::try_from(port).ok()) + }) + .collect::>(); + for port in required_ports { + if existing_ports.contains(&port) { + continue; + } + ports.push( + serde_yaml::to_value(json!({ + "protocol": "TCP", + "port": port, + })) + .into_diagnostic() + .wrap_err("failed to serialize site controller router ingress port")?, + ); + } + } else { ingress.push( serde_yaml::to_value(json!({ "from": [{ @@ -407,10 +504,10 @@ fn ensure_kubernetes_router_allows_site_controller_control(artifact_root: &Path) "matchLabels": controller_selector, } }], - "ports": [{ + "ports": required_ports.into_iter().map(|port| json!({ "protocol": "TCP", - "port": KUBERNETES_ROUTER_CONTROL_PORT, - }] + "port": port, + })).collect::>() })) .into_diagnostic() .wrap_err("failed to serialize site controller router ingress rule")?, diff --git a/runtime/site-controller/src/default_runtime/site_artifacts.rs b/runtime/site-controller/src/default_runtime/site_artifacts.rs index 0a3bcd66..adc3e836 100644 --- a/runtime/site-controller/src/default_runtime/site_artifacts.rs +++ b/runtime/site-controller/src/default_runtime/site_artifacts.rs @@ -1484,16 +1484,6 @@ pub fn set_compose_router_published_mesh_port(artifact_root: &Path, host_port: u set_compose_router_published_port(artifact_root, "0.0.0.0", host_port, 24000, false) } -pub fn add_compose_router_published_route_ports( - artifact_root: &Path, - route_ports: &[u16], -) -> Result<()> { - for port in route_ports { - set_compose_router_published_port(artifact_root, "127.0.0.1", *port, *port, true)?; - } - Ok(()) -} - fn set_compose_router_published_port( artifact_root: &Path, host_ip: &str, diff --git a/runtime/site-controller/src/default_runtime/site_runtime_support.rs b/runtime/site-controller/src/default_runtime/site_runtime_support.rs index 9de31ab0..0288ece7 100644 --- a/runtime/site-controller/src/default_runtime/site_runtime_support.rs +++ b/runtime/site-controller/src/default_runtime/site_runtime_support.rs @@ -76,7 +76,6 @@ pub(super) fn site_supervisor_plan_for_site_runtime( observability_endpoint: plan.observability_endpoint.clone(), site_controller_plan_path: None, site_controller_url: None, - controller_route_ports: Vec::new(), launch_env: plan.launch_env.clone(), } } @@ -230,11 +229,42 @@ pub fn observability_endpoint_for_site(kind: SiteKind, endpoint: &str) -> Result Ok(url.to_string()) } -pub(crate) fn reserve_loopback_port() -> Result { - let listener = TcpListener::bind(("127.0.0.1", 0)) - .into_diagnostic() - .wrap_err("failed to allocate a loopback port")?; - Ok(listener.local_addr().into_diagnostic()?.port()) +pub fn reserve_loopback_port() -> Result { + const LOOPBACK_PORT_RANGE_START: u16 = 30000; + const LOOPBACK_PORT_RANGE_END: u16 = 60000; + static RESERVED_LOOPBACK_PORTS: OnceLock>> = OnceLock::new(); + + let reserved = RESERVED_LOOPBACK_PORTS.get_or_init(|| std::sync::Mutex::new(BTreeSet::new())); + let mut reserved = reserved + .lock() + .expect("loopback port allocator should not be poisoned"); + let span = u32::from(LOOPBACK_PORT_RANGE_END - LOOPBACK_PORT_RANGE_START); + let mut next = + LOOPBACK_PORT_RANGE_START + (std::process::id() % span) as u16 + reserved.len() as u16; + for _ in 0..usize::from(LOOPBACK_PORT_RANGE_END - LOOPBACK_PORT_RANGE_START) { + if next >= LOOPBACK_PORT_RANGE_END { + next = LOOPBACK_PORT_RANGE_START; + } + let port = next; + next += 1; + if reserved.contains(&port) { + continue; + } + match TcpListener::bind(("127.0.0.1", port)) { + Ok(listener) => { + drop(listener); + reserved.insert(port); + return Ok(port); + } + Err(_) => continue, + } + } + + Err(miette::miette!( + "failed to allocate a unique loopback port in {}-{}", + LOOPBACK_PORT_RANGE_START, + LOOPBACK_PORT_RANGE_END - 1 + )) } pub(super) fn site_supervisor_plan_path(site_state_root: &Path) -> PathBuf { @@ -410,6 +440,9 @@ pub(super) fn kubectl_command(context: Option<&str>) -> Command { } pub(super) fn ensure_kubernetes_namespace(plan: &SiteSupervisorPlan) -> Result<()> { + if !should_prepare_kubernetes_namespace(plan, is_in_cluster_kubernetes_runtime()) { + return Ok(()); + } let namespace = required_str(plan.kubernetes_namespace.as_deref(), "kubernetes namespace")?; let deadline = Instant::now() + Duration::from_secs(60); let context = plan.context.as_deref(); @@ -472,6 +505,17 @@ pub(super) fn ensure_kubernetes_namespace(plan: &SiteSupervisorPlan) -> Result<( } } +fn should_prepare_kubernetes_namespace( + plan: &SiteSupervisorPlan, + in_cluster_runtime: bool, +) -> bool { + plan.context.is_some() || !in_cluster_runtime +} + +fn is_in_cluster_kubernetes_runtime() -> bool { + env::var_os("KUBERNETES_SERVICE_HOST").is_some() +} + pub(super) fn ensure_kubernetes_workloads_ready(plan: &SiteSupervisorPlan) -> Result<()> { let namespace = required_str(plan.kubernetes_namespace.as_deref(), "kubernetes namespace")?; let context = plan.context.as_deref(); @@ -1045,6 +1089,14 @@ pub(crate) fn host_service_host_for_consumer(consumer_kind: SiteKind) -> String } } +pub fn site_controller_peer_router_url(controller_site_kind: SiteKind, route_port: u16) -> String { + let host = match controller_site_kind { + SiteKind::Direct | SiteKind::Vm => "127.0.0.1", + SiteKind::Compose | SiteKind::Kubernetes => KUBERNETES_ROUTER_COMPONENT_NAME, + }; + format!("http://{host}:{route_port}") +} + pub(super) fn consumer_needs_host_wide_listener(consumer_kind: SiteKind) -> bool { matches!(consumer_kind, SiteKind::Compose | SiteKind::Kubernetes) } @@ -1222,8 +1274,43 @@ pub(super) fn resolve_desktop_container_host_ip() -> Option { #[cfg(test)] mod tests { + use std::collections::BTreeMap; + + use amber_compiler::run_plan::SiteKind; + #[cfg(unix)] use super::parse_process_status_code; + use super::{ + SITE_PLAN_SCHEMA, SITE_PLAN_VERSION, SiteSupervisorPlan, + should_prepare_kubernetes_namespace, + }; + + fn kubernetes_supervisor_plan(context: Option<&str>) -> SiteSupervisorPlan { + SiteSupervisorPlan { + schema: SITE_PLAN_SCHEMA.to_string(), + version: SITE_PLAN_VERSION, + run_id: "run-1".to_string(), + mesh_scope: "test".to_string(), + run_root: "/tmp/run".to_string(), + coordinator_pid: 0, + site_id: "kind_local".to_string(), + kind: SiteKind::Kubernetes, + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), + storage_root: None, + runtime_root: None, + router_mesh_port: None, + compose_project: None, + kubernetes_namespace: Some("amber-test-kind-local".to_string()), + context: context.map(str::to_string), + port_forward_mesh_port: None, + port_forward_control_port: None, + observability_endpoint: None, + site_controller_plan_path: None, + site_controller_url: None, + launch_env: BTreeMap::new(), + } + } #[cfg(unix)] #[test] @@ -1232,4 +1319,23 @@ mod tests { assert_eq!(parse_process_status_code("z\n"), Some('Z')); assert_eq!(parse_process_status_code(""), None); } + + #[test] + fn in_cluster_kubernetes_controller_skips_namespace_bootstrap() { + let plan = kubernetes_supervisor_plan(None); + assert!( + !should_prepare_kubernetes_namespace(&plan, true), + "in-cluster site controllers should not perform cluster-scoped namespace probes" + ); + } + + #[test] + fn external_kubernetes_supervisor_still_prepares_namespace() { + let plan = kubernetes_supervisor_plan(Some("kind-test")); + assert!( + should_prepare_kubernetes_namespace(&plan, true), + "external site supervisors still need to prepare the namespace before applying \ + artifacts" + ); + } } diff --git a/runtime/site-controller/src/lib.rs b/runtime/site-controller/src/lib.rs index 6cc19a67..cdcf5430 100644 --- a/runtime/site-controller/src/lib.rs +++ b/runtime/site-controller/src/lib.rs @@ -78,13 +78,13 @@ mod tests; pub use self::{ default_runtime::{ - SiteControllerPeerRouterRoute, add_compose_router_published_route_ports, - assign_compose_egress_network_subnets, cleanup_dynamic_site_children, - host_service_bind_addr_for_consumer, inject_compose_site_controller, - inject_kubernetes_site_controller, inject_site_controller_peer_router_routes, - observability_endpoint_for_site, prepare_kubernetes_artifact_namespace, + SiteControllerPeerRouterRoute, assign_compose_egress_network_subnets, + cleanup_dynamic_site_children, host_service_bind_addr_for_consumer, + inject_compose_site_controller, inject_kubernetes_site_controller, + inject_site_controller_peer_router_routes, observability_endpoint_for_site, + prepare_kubernetes_artifact_namespace, reserve_loopback_port, router_mesh_addr_for_consumer, set_compose_router_published_mesh_port, - set_site_artifact_mesh_identity_seed, walk_files, + set_site_artifact_mesh_identity_seed, site_controller_peer_router_url, walk_files, }, runtime_api::{ DesiredExportPeerOverlay, DesiredExternalSlotOverlay, DesiredRouteOverlay, LaunchedSite, diff --git a/runtime/site-controller/src/orchestration.rs b/runtime/site-controller/src/orchestration.rs index a1d58a43..53fb35c6 100644 --- a/runtime/site-controller/src/orchestration.rs +++ b/runtime/site-controller/src/orchestration.rs @@ -140,6 +140,17 @@ pub(crate) struct RevokeDynamicCapabilityOriginOverlaysRequest { pub(crate) overlay_ids: Vec, } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct RemoteChildRuntimeRequest { + pub(crate) state: FrameworkControlState, + pub(crate) child: LiveChildRecord, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct RemoteChildRollbackRequest { + pub(crate) child_id: u64, +} + pub(super) fn control_state_step_error( step: &str, err: impl std::fmt::Display, @@ -762,6 +773,45 @@ fn peer_site_router_url( }) } +const PEER_CONTROLLER_REQUEST_RETRY_TIMEOUT: Duration = Duration::from_secs(30); +const PEER_CONTROLLER_REQUEST_RETRY_DELAY: Duration = Duration::from_millis(250); + +fn should_retry_peer_controller_status(status: StatusCode) -> bool { + matches!( + status, + StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE | StatusCode::GATEWAY_TIMEOUT + ) +} + +async fn post_json_with_retry( + client: &ReqwestClient, + url: &str, + body: &TReq, +) -> std::result::Result { + let deadline = tokio::time::Instant::now() + PEER_CONTROLLER_REQUEST_RETRY_TIMEOUT; + loop { + match client + .post(url) + .header(super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER, "1") + .json(body) + .send() + .await + { + Ok(response) + if should_retry_peer_controller_status(response.status()) + && tokio::time::Instant::now() < deadline => + { + tokio::time::sleep(PEER_CONTROLLER_REQUEST_RETRY_DELAY).await; + } + Ok(response) => return Ok(response), + Err(err) if tokio::time::Instant::now() < deadline => { + tokio::time::sleep(PEER_CONTROLLER_REQUEST_RETRY_DELAY).await; + } + Err(err) => return Err(err), + } + } +} + async fn peer_controller_post_json_via_router Deserialize<'de>>( app: &ControlStateApp, site_id: &str, @@ -770,18 +820,17 @@ async fn peer_controller_post_json_via_router D code: ProtocolErrorCode, action: &str, ) -> std::result::Result { - let response = app - .client - .post(format!( + let response = post_json_with_retry( + &app.client, + &format!( "{}{}", peer_site_router_url(app, site_id)?.trim_end_matches('/'), path - )) - .header(super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER, "1") - .json(body) - .send() - .await - .map_err(|err| controller_protocol_error(code, site_id, action, err))?; + ), + body, + ) + .await + .map_err(|err| controller_protocol_error(code, site_id, action, err))?; if response.status().is_success() { return response .json() @@ -812,18 +861,17 @@ async fn peer_controller_post_no_content_via_router( code: ProtocolErrorCode, action: &str, ) -> std::result::Result<(), ProtocolErrorResponse> { - let response = app - .client - .post(format!( + let response = post_json_with_retry( + &app.client, + &format!( "{}{}", peer_site_router_url(app, site_id)?.trim_end_matches('/'), path - )) - .header(super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER, "1") - .json(body) - .send() - .await - .map_err(|err| controller_protocol_error(code, site_id, action, err))?; + ), + body, + ) + .await + .map_err(|err| controller_protocol_error(code, site_id, action, err))?; if response.status().is_success() { return Ok(()); } @@ -1307,27 +1355,38 @@ pub(super) async fn prepare_child_on_site( app: &ControlStateApp, state: &FrameworkControlState, child: &LiveChildRecord, + site_id: &str, ) -> std::result::Result<(), ProtocolErrorResponse> { - let site_id = child_runtime_site_id(child)?; if site_id != app.controller_plan.site_id { - return Err(controller_protocol_error( + return peer_controller_post_no_content_via_router( + app, + site_id, + "/v1/internal/children/prepare", + &RemoteChildRuntimeRequest { + state: state.clone(), + child: child.clone(), + }, ProtocolErrorCode::PrepareFailed, - &site_id, "prepare child", - format!( - "dynamic child `{}` targeted site `{}` but controller `{}` only creates local \ - children", - child.name, site_id, app.controller_plan.site_id - ), - )); + ) + .await; } + prepare_child_on_local_site(app, state, child, site_id).await +} + +pub(super) async fn prepare_child_on_local_site( + app: &ControlStateApp, + state: &FrameworkControlState, + child: &LiveChildRecord, + site_id: &str, +) -> std::result::Result<(), ProtocolErrorResponse> { app.runtime .prepare_child(&app.controller_plan, state.clone(), child.clone()) .await .map_err(|err| { controller_protocol_error( ProtocolErrorCode::PrepareFailed, - &site_id, + site_id, "prepare child", err, ) @@ -1338,27 +1397,38 @@ pub(super) async fn publish_child_on_site( app: &ControlStateApp, state: &FrameworkControlState, child: &LiveChildRecord, + site_id: &str, ) -> std::result::Result<(), ProtocolErrorResponse> { - let site_id = child_runtime_site_id(child)?; if site_id != app.controller_plan.site_id { - return Err(controller_protocol_error( + return peer_controller_post_no_content_via_router( + app, + site_id, + "/v1/internal/children/publish", + &RemoteChildRuntimeRequest { + state: state.clone(), + child: child.clone(), + }, ProtocolErrorCode::PublishFailed, - &site_id, "publish child", - format!( - "dynamic child `{}` targeted site `{}` but controller `{}` only publishes local \ - children", - child.name, site_id, app.controller_plan.site_id - ), - )); + ) + .await; } + publish_child_on_local_site(app, state, child, site_id).await +} + +pub(super) async fn publish_child_on_local_site( + app: &ControlStateApp, + state: &FrameworkControlState, + child: &LiveChildRecord, + site_id: &str, +) -> std::result::Result<(), ProtocolErrorResponse> { app.runtime .publish_child(&app.controller_plan, state.clone(), child.clone()) .await .map_err(|err| { controller_protocol_error( ProtocolErrorCode::PublishFailed, - &site_id, + site_id, "publish child", err, ) @@ -1376,12 +1446,13 @@ pub(super) async fn publish_child_on_site_with_prepare_retry( app: &ControlStateApp, state: &FrameworkControlState, child: &LiveChildRecord, + site_id: &str, ) -> std::result::Result<(), ProtocolErrorResponse> { - match publish_child_on_site(app, state, child).await { + match publish_child_on_site(app, state, child, site_id).await { Ok(()) => Ok(()), Err(err) if site_controller_child_needs_prepare(&err, child.child_id) => { - prepare_child_on_site(app, state, child).await?; - publish_child_on_site(app, state, child).await + prepare_child_on_site(app, state, child, site_id).await?; + publish_child_on_site(app, state, child, site_id).await } Err(err) => Err(err), } @@ -1393,11 +1464,16 @@ pub(super) async fn rollback_child_on_site( site_id: &str, ) -> Result<()> { if site_id != app.controller_plan.site_id { - return Err(miette::miette!( - "dynamic child plan targeted site `{site_id}` but controller `{}` only rolls back \ - local children", - app.controller_plan.site_id - )); + return peer_controller_post_no_content_via_router( + app, + site_id, + "/v1/internal/children/rollback", + &RemoteChildRollbackRequest { child_id }, + ProtocolErrorCode::ControlStateUnavailable, + "rollback child", + ) + .await + .map_err(|err| miette::miette!(err.message)); } app.runtime .rollback_child(&app.controller_plan, child_id) @@ -1408,33 +1484,77 @@ pub(super) async fn destroy_child_on_site( app: &ControlStateApp, state: &FrameworkControlState, child: &LiveChildRecord, + site_id: &str, ) -> std::result::Result<(), ProtocolErrorResponse> { - let site_id = child_runtime_site_id(child)?; if site_id != app.controller_plan.site_id { - return Err(controller_protocol_error( + return peer_controller_post_no_content_via_router( + app, + site_id, + "/v1/internal/children/destroy", + &RemoteChildRuntimeRequest { + state: state.clone(), + child: child.clone(), + }, ProtocolErrorCode::ControlStateUnavailable, - &site_id, "destroy child", - format!( - "dynamic child `{}` targeted site `{site_id}` but controller `{}` only destroys \ - local children", - child.name, app.controller_plan.site_id - ), - )); + ) + .await; } + destroy_child_on_local_site(app, state, child, site_id).await +} + +pub(super) async fn destroy_child_on_local_site( + app: &ControlStateApp, + state: &FrameworkControlState, + child: &LiveChildRecord, + site_id: &str, +) -> std::result::Result<(), ProtocolErrorResponse> { app.runtime .destroy_child(&app.controller_plan, state.clone(), child.clone()) .await .map_err(|err| { controller_protocol_error( ProtocolErrorCode::ControlStateUnavailable, - &site_id, + site_id, "destroy child", err, ) }) } +pub(super) async fn prepare_child_on_sites( + app: &ControlStateApp, + state: &FrameworkControlState, + child: &LiveChildRecord, +) -> std::result::Result<(), ProtocolErrorResponse> { + for site_id in child_runtime_site_ids(child)? { + prepare_child_on_site(app, state, child, &site_id).await?; + } + Ok(()) +} + +pub(super) async fn publish_child_on_sites_with_prepare_retry( + app: &ControlStateApp, + state: &FrameworkControlState, + child: &LiveChildRecord, +) -> std::result::Result<(), ProtocolErrorResponse> { + for site_id in child_runtime_site_ids(child)? { + publish_child_on_site_with_prepare_retry(app, state, child, &site_id).await?; + } + Ok(()) +} + +pub(super) async fn destroy_child_on_sites( + app: &ControlStateApp, + state: &FrameworkControlState, + child: &LiveChildRecord, +) -> std::result::Result<(), ProtocolErrorResponse> { + for site_id in child_runtime_site_ids(child)? { + destroy_child_on_site(app, state, child, &site_id).await?; + } + Ok(()) +} + pub(super) async fn publish_external_slot_overlay( app: &ControlStateApp, child: &LiveChildRecord, @@ -1747,16 +1867,6 @@ pub(super) async fn clear_export_peer_overlay( } } -pub(super) async fn publish_child_overlays( - app: &ControlStateApp, - child: &LiveChildRecord, -) -> std::result::Result<(), ProtocolErrorResponse> { - for link in child_link_records(child) { - publish_link_overlays(app, child, &link).await?; - } - Ok(()) -} - pub(super) async fn retract_child_overlays( app: &ControlStateApp, child: &LiveChildRecord, @@ -1846,13 +1956,13 @@ pub(super) fn cloned_child_record( }) } -pub(super) async fn rollback_prepared_sites( - app: &ControlStateApp, - child_id: u64, - prepared: bool, -) -> Result<()> { - if prepared { - rollback_child_on_site(app, child_id, &app.controller_plan.site_id).await?; +pub(super) async fn rollback_prepared_sites(app: &ControlStateApp, child_id: u64) -> Result<()> { + let child = { + let state = app.control_state.lock().await; + cloned_child_record(&state, child_id).map_err(|err| miette::miette!(err.message))? + }; + for site_id in child_runtime_site_ids(&child).map_err(|err| miette::miette!(err.message))? { + rollback_child_on_site(app, child_id, &site_id).await?; } Ok(()) } @@ -1885,24 +1995,8 @@ pub(super) async fn continue_create_committed_hidden( }; let state = app.control_state.lock().await.clone(); - let published_links = child_link_records(&child); - for link in &published_links { - publish_link_overlays(app, &child, link).await?; - } - if let Err(err) = publish_child_on_site_with_prepare_retry(app, &state, &child).await { - let mut cleanup_error = None; - for link in published_links.iter().rev() { - if let Err(retract_err) = retract_link_overlays(app, &child, link).await - && cleanup_error.is_none() - { - cleanup_error = Some(retract_err); - } - } - if let Err(destroy_err) = destroy_child_on_site(app, &state, &child).await - && cleanup_error.is_none() - { - cleanup_error = Some(destroy_err); - } + if let Err(err) = publish_child_on_sites_with_prepare_retry(app, &state, &child).await { + let cleanup_error = destroy_child_on_sites(app, &state, &child).await.err(); if let Some(cleanup_error) = cleanup_error { return Err(protocol_error( ProtocolErrorCode::PublishFailed, @@ -1920,7 +2014,47 @@ pub(super) async fn continue_create_committed_hidden( } return Err(err); } - publish_child_overlays(app, &child).await?; + let published_links = child_link_records(&child); + let mut applied_links = Vec::new(); + for link in &published_links { + if let Err(err) = publish_link_overlays(app, &child, link).await { + let mut cleanup_error = None; + for published_link in applied_links.iter().rev() { + if let Err(retract_err) = retract_link_overlays(app, &child, published_link).await + && cleanup_error.is_none() + { + cleanup_error = Some(retract_err); + } + } + if let Err(destroy_err) = destroy_child_on_sites(app, &state, &child).await + && cleanup_error.is_none() + { + cleanup_error = Some(destroy_err); + } + if let Some(cleanup_error) = cleanup_error { + return Err(protocol_error( + ProtocolErrorCode::PublishFailed, + &format!("{}; cleanup failed: {}", err.message, cleanup_error.message), + )); + } + let mut state = app.control_state.lock().await; + if child_record_location(&state, child.child_id).is_ok() { + let tx_id = child_create_tx_id(&state, child.child_id)?; + persist_control_state_update( + &mut state, + &app.state_path, + "create_aborted", + |state| { + append_journal_entry(state, tx_id, &child, ChildState::CreateAborted); + remove_child_record(state, child.child_id)?; + Ok(()) + }, + )?; + } + return Err(err); + } + applied_links.push(link.clone()); + } let mut state = app.control_state.lock().await; let child = cloned_child_record(&state, child_id)?; @@ -1951,7 +2085,7 @@ pub(super) async fn continue_destroy_retracted( child }; let state = app.control_state.lock().await.clone(); - destroy_child_on_site(app, &state, &child).await?; + destroy_child_on_sites(app, &state, &child).await?; let mut state = app.control_state.lock().await; let child = cloned_child_record(&state, child_id)?; @@ -2024,8 +2158,8 @@ pub(super) async fn execute_create_child( reconcile_local_framework_routes(app).await?; let state = app.control_state.lock().await.clone(); - if let Err(err) = prepare_child_on_site(app, &state, &child).await { - let rollback_err = rollback_prepared_sites(app, child.child_id, false).await; + if let Err(err) = prepare_child_on_sites(app, &state, &child).await { + let rollback_err = rollback_prepared_sites(app, child.child_id).await; let should_reconcile = { let mut state = app.control_state.lock().await; if state @@ -2182,7 +2316,7 @@ pub(super) async fn recover_control_state(app: &ControlStateApp) -> Result<()> { } } ChildState::CreatePrepared => { - rollback_prepared_sites(app, child.child_id, true) + rollback_prepared_sites(app, child.child_id) .await .wrap_err_with(|| { format!( @@ -2234,3 +2368,56 @@ pub(super) async fn recover_control_state(app: &ControlStateApp) -> Result<()> { .map_err(|err| miette::miette!(err.message))?; Ok(()) } + +#[cfg(test)] +mod tests { + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + + use axum::{Json, Router, extract::State, response::Response, routing::post}; + use serde_json::json; + use tokio::net::TcpListener; + + use super::*; + + #[tokio::test] + async fn post_json_with_retry_tolerates_transient_service_unavailable() { + async fn handler(State(attempts): State>) -> Response { + let attempt = attempts.fetch_add(1, Ordering::SeqCst); + if attempt < 2 { + return (StatusCode::SERVICE_UNAVAILABLE, Json(json!({"ok": false}))) + .into_response(); + } + (StatusCode::OK, Json(json!({"ok": true}))).into_response() + } + + let attempts = Arc::new(AtomicUsize::new(0)); + let app = Router::new() + .route("/retry", post(handler)) + .with_state(attempts.clone()); + let listener = TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("listener should bind"); + let addr = listener.local_addr().expect("listener addr"); + let serve = tokio::spawn(async move { + axum::serve(listener, app) + .await + .expect("test server should run"); + }); + + let response = post_json_with_retry( + &ReqwestClient::new(), + &format!("http://{addr}/retry"), + &json!({"hello": "world"}), + ) + .await + .expect("request should eventually succeed"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(attempts.load(Ordering::SeqCst), 3); + + serve.abort(); + let _ = serve.await; + } +} diff --git a/runtime/site-controller/src/planner.rs b/runtime/site-controller/src/planner.rs index 3794c89f..694eb36a 100644 --- a/runtime/site-controller/src/planner.rs +++ b/runtime/site-controller/src/planner.rs @@ -137,23 +137,6 @@ pub(super) fn rebuild_live_child_runtime_metadata( &format!("child placement could not be satisfied: {err}"), ) })?; - let fragment_sites = fragment - .components - .iter() - .filter_map(|component| planned.assignments.get(component.moniker.as_str())) - .cloned() - .collect::>(); - if fragment_sites.len() > 1 { - return Err(protocol_error( - ProtocolErrorCode::PlacementUnsatisfied, - &format!( - "dynamic child `{}` spans multiple sites ({}) but site controllers only create \ - children within a single site", - child.name, - fragment_sites.into_iter().collect::>().join(", ") - ), - )); - } child.assignments = planned.assignments; child.overlays = dynamic_overlay_records(&planned.incident_links, fragment); let mut live_assignments = existing_assignments.clone(); @@ -1490,9 +1473,9 @@ fn build_local_child_runtime_spec_from_site_plan( }) } -pub(super) fn child_runtime_site_id( +pub(super) fn child_runtime_site_ids( child: &LiveChildRecord, -) -> std::result::Result { +) -> std::result::Result, ProtocolErrorResponse> { let fragment = child.fragment.as_ref().ok_or_else(|| { protocol_error( ProtocolErrorCode::ControlStateUnavailable, @@ -1508,18 +1491,27 @@ pub(super) fn child_runtime_site_id( .filter_map(|component| child.assignments.get(component.moniker.as_str())) .cloned() .collect::>(); - match site_ids.len() { - 1 => Ok(site_ids - .into_iter() - .next() - .expect("single child site id should be present")), - 0 => Err(protocol_error( + if site_ids.is_empty() { + return Err(protocol_error( ProtocolErrorCode::ControlStateUnavailable, &format!( "dynamic child `{}` has no site assignment for any fragment component", child.name ), - )), + )); + } + Ok(site_ids.into_iter().collect()) +} + +pub(super) fn child_runtime_site_id( + child: &LiveChildRecord, +) -> std::result::Result { + let site_ids = child_runtime_site_ids(child)?; + match site_ids.len() { + 1 => Ok(site_ids + .into_iter() + .next() + .expect("single child site id should be present")), _ => Err(protocol_error( ProtocolErrorCode::PlacementUnsatisfied, &format!( diff --git a/runtime/site-controller/src/site_controller.rs b/runtime/site-controller/src/site_controller.rs index a1cf83f1..a5ab3f53 100644 --- a/runtime/site-controller/src/site_controller.rs +++ b/runtime/site-controller/src/site_controller.rs @@ -35,12 +35,15 @@ use super::{ orchestration::{ ClearExportPeerOverlayRequest, ClearExternalSlotOverlayRequest, ProtocolApiError, PublishExportPeerOverlayRequest, PublishExternalSlotOverlayRequest, - ResolveExternalLinkUrlRequest, ResolveExternalLinkUrlResponse, - RevokeDynamicCapabilityOriginOverlaysRequest, clear_export_peer_overlay_local, - clear_external_slot_overlay_local, execute_create_child, execute_destroy_child, + RemoteChildRollbackRequest, RemoteChildRuntimeRequest, ResolveExternalLinkUrlRequest, + ResolveExternalLinkUrlResponse, RevokeDynamicCapabilityOriginOverlaysRequest, + clear_export_peer_overlay_local, clear_external_slot_overlay_local, + destroy_child_on_local_site, execute_create_child, execute_destroy_child, + prepare_child_on_local_site, publish_child_on_local_site, publish_dynamic_capability_origin_local, publish_export_peer_overlay_local, publish_external_slot_overlay_local, recover_control_state, resolve_external_link_url_local, revoke_dynamic_capability_origin_overlays_local, + rollback_child_on_site, }, planner::{ ControlStateApp, LocalDynamicCapabilityOriginApp, SiteControllerApp, protocol_error, @@ -159,6 +162,22 @@ pub(crate) fn site_controller_router(app_state: SiteControllerApp) -> Router { "/v1/internal/link-overlays/external-url", post(resolve_external_link_url), ) + .route( + "/v1/internal/children/prepare", + post(prepare_child_runtime_route), + ) + .route( + "/v1/internal/children/publish", + post(publish_child_runtime_route), + ) + .route( + "/v1/internal/children/rollback", + post(rollback_child_runtime_route), + ) + .route( + "/v1/internal/children/destroy", + post(destroy_child_runtime_route), + ) .route( "/v1/internal/link-overlays/external-slot/publish", post(publish_external_slot_overlay_route), @@ -1112,6 +1131,62 @@ async fn resolve_external_link_url( )) } +async fn prepare_child_runtime_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize_local_controller_request(&app, &headers)?; + ensure_controller_ready(&app)?; + let site_id = app.control.controller_plan.site_id.clone(); + prepare_child_on_local_site(&app.control, &request.state, &request.child, &site_id) + .await + .map_err(ProtocolApiError::from)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn publish_child_runtime_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize_local_controller_request(&app, &headers)?; + ensure_controller_ready(&app)?; + let site_id = app.control.controller_plan.site_id.clone(); + publish_child_on_local_site(&app.control, &request.state, &request.child, &site_id) + .await + .map_err(ProtocolApiError::from)?; + Ok(StatusCode::NO_CONTENT) +} + +async fn rollback_child_runtime_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize_local_controller_request(&app, &headers)?; + ensure_controller_ready(&app)?; + let site_id = app.control.controller_plan.site_id.clone(); + rollback_child_on_site(&app.control, request.child_id, &site_id) + .await + .map_err(|err| ProtocolApiError::control_state_unavailable(err.to_string()))?; + Ok(StatusCode::NO_CONTENT) +} + +async fn destroy_child_runtime_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result { + authorize_local_controller_request(&app, &headers)?; + ensure_controller_ready(&app)?; + let site_id = app.control.controller_plan.site_id.clone(); + destroy_child_on_local_site(&app.control, &request.state, &request.child, &site_id) + .await + .map_err(ProtocolApiError::from)?; + Ok(StatusCode::NO_CONTENT) +} + async fn publish_external_slot_overlay_route( State(app): State, headers: HeaderMap, diff --git a/runtime/site-controller/src/tests.rs b/runtime/site-controller/src/tests.rs index 9e56f5aa..2e0b8216 100644 --- a/runtime/site-controller/src/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -246,6 +246,7 @@ fn test_runtime() -> SharedSiteControllerRuntime { type DestroyCalls = Arc>>; type PublishCalls = Arc>>; +type ResolveCalls = Arc>>; #[derive(Clone, Default)] struct FailingPublishRuntime { @@ -511,6 +512,167 @@ impl SiteControllerRuntime for RecordingPublishRuntime { } } +#[derive(Clone, Default)] +struct PublishBeforeResolveRuntime { + publish_calls: PublishCalls, + resolve_calls: ResolveCalls, +} + +impl SiteControllerRuntime for PublishBeforeResolveRuntime { + fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()> { + TestSiteControllerRuntime.cleanup() + } + + fn resolve_link_external_url<'a>( + &'a self, + provider: &'a LaunchedSite, + provider_output_dir: &'a Path, + link: &'a amber_compiler::run_plan::RunLink, + consumer_kind: SiteKind, + run_root: &'a Path, + ) -> SiteControllerRuntimeFuture<'a, String> { + let publish_calls = self.publish_calls.clone(); + let resolve_calls = self.resolve_calls.clone(); + Box::pin(async move { + resolve_calls + .lock() + .expect("resolve call log mutex should lock") + .push(link.export_name.clone()); + if publish_calls + .lock() + .expect("publish call log mutex should lock") + .is_empty() + { + return Err(miette::miette!( + "resolve_link_external_url ran before publish_child" + )); + } + TestSiteControllerRuntime + .resolve_link_external_url( + provider, + provider_output_dir, + link, + consumer_kind, + run_root, + ) + .await + }) + } + + fn prepare_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn publish_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + child: LiveChildRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + let publish_calls = self.publish_calls.clone(); + Box::pin(async move { + let site_id = + child_runtime_site_id(&child).unwrap_or_else(|_| "multi-site".to_string()); + publish_calls + .lock() + .expect("publish call log mutex should lock") + .push((child.child_id, site_id)); + Ok(()) + }) + } + + fn rollback_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _child_id: u64, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn destroy_child<'a>( + &'a self, + _plan: &'a SiteControllerPlan, + _state: FrameworkControlState, + _child: LiveChildRecord, + ) -> SiteControllerRuntimeFuture<'a, ()> { + Box::pin(async { Ok(()) }) + } + + fn collect_live_component_runtime_metadata( + &self, + plan: &SiteControllerRuntimePlan, + ) -> miette::Result> { + TestSiteControllerRuntime.collect_live_component_runtime_metadata(plan) + } + + fn load_live_site_router_mesh_config( + &self, + plan: &SiteControllerRuntimePlan, + ) -> miette::Result { + TestSiteControllerRuntime.load_live_site_router_mesh_config(plan) + } + + fn router_mesh_addr_for_consumer( + &self, + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> miette::Result { + TestSiteControllerRuntime.router_mesh_addr_for_consumer( + provider_kind, + consumer_kind, + router_mesh_addr, + ) + } + + fn update_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExternalSlotOverlay, + ) -> miette::Result<()> { + TestSiteControllerRuntime.update_desired_overlay_for_consumer( + site_state_root, + overlay_id, + overlay, + ) + } + + fn update_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + overlay: DesiredExportPeerOverlay, + ) -> miette::Result<()> { + TestSiteControllerRuntime.update_desired_overlay_for_provider( + site_state_root, + overlay_id, + overlay, + ) + } + + fn clear_desired_overlay_for_consumer( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> miette::Result<()> { + TestSiteControllerRuntime.clear_desired_overlay_for_consumer(site_state_root, overlay_id) + } + + fn clear_desired_overlay_for_provider( + &self, + site_state_root: &Path, + overlay_id: &str, + ) -> miette::Result<()> { + TestSiteControllerRuntime.clear_desired_overlay_for_provider(site_state_root, overlay_id) + } +} + #[derive(Clone, Default)] struct FailingRollbackRuntime; @@ -3689,12 +3851,17 @@ async fn prepare_child_on_site_rejects_nonlocal_site_plan() { let app = test_control_state_app(&dir, state, state_path); let mut child = empty_live_child(1, "remote", 1, ChildState::CreatePrepared); child.assignments = BTreeMap::from([("/remote".to_string(), "other-site".to_string())]); - let err = prepare_child_on_site(&app, &app.control_state.lock().await.clone(), &child) - .await - .expect_err("nonlocal children should be rejected"); - assert_eq!(err.code, ProtocolErrorCode::PrepareFailed); + let err = prepare_child_on_site( + &app, + &app.control_state.lock().await.clone(), + &child, + "other-site", + ) + .await + .expect_err("nonlocal children without a router path should fail"); + assert_eq!(err.code, ProtocolErrorCode::OriginUnavailable); assert!( - err.message.contains("only creates local children"), + err.message.contains("has no router forward path"), "unexpected error: {}", err.message ); @@ -5757,6 +5924,263 @@ async fn capability_instance_auth_and_snapshot_scope_are_enforced() { assert_eq!(snapshot_err.code, ProtocolErrorCode::ScopeNotAllowed); } +#[tokio::test] +async fn create_child_publishes_sites_before_resolving_cross_site_link_overlays() { + let dir = TempDir::new().expect("temp dir"); + let root_path = dir.path().join("root.json5"); + write_file( + &root_path, + r#" + { + manifest_version: "0.3.0", + program: { path: "/bin/echo", args: ["root"] } + } + "#, + ); + + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "compose_local".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + }, + ), + ( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_local".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([("/job-1/consumer".to_string(), "compose_local".to_string())]), + dynamic_capabilities: None, + framework_children: None, + }; + let mut state = compile_control_state_with_placement(&root_path, Some(&placement)).await; + let state_path = dir.path().join("control-state.json"); + let root_authority = state.base_scenario.root; + let child_root_id = 10_001usize; + let provider_id = 10_002usize; + let consumer_id = 10_003usize; + let link = RunLink { + provider_site: "direct_local".to_string(), + consumer_site: "compose_local".to_string(), + provider_component: "/job-1/provider".to_string(), + provide: "http".to_string(), + consumer_component: "/job-1/consumer".to_string(), + slot: "upstream".to_string(), + weak: false, + protocol: amber_manifest::NetworkProtocol::Http, + export_name: "amber_export_test".to_string(), + external_slot_name: "amber_link_test".to_string(), + }; + let child = LiveChildRecord { + child_id: 1, + authority_realm_id: root_authority, + name: "job-1".to_string(), + state: ChildState::CreateCommittedHidden, + template_name: Some("worker".to_string()), + selected_manifest_catalog_key: None, + fragment: Some(LiveScenarioFragment { + root_component_id: child_root_id, + components: vec![ + ComponentIr { + id: child_root_id, + moniker: "/job-1".to_string(), + parent: Some(root_authority), + children: vec![provider_id, consumer_id], + resolved_url: Some("file:///tmp/job-1.json5".to_string()), + digest: amber_manifest::ManifestDigest::new([0; 32]), + config: None, + config_schema: None, + program: None, + slots: BTreeMap::new(), + provides: BTreeMap::new(), + exports: BTreeMap::new(), + resources: BTreeMap::new(), + child_templates: BTreeMap::new(), + metadata: None, + }, + ComponentIr { + id: provider_id, + moniker: "/job-1/provider".to_string(), + parent: Some(child_root_id), + children: Vec::new(), + resolved_url: Some("file:///tmp/job-1-provider.json5".to_string()), + digest: amber_manifest::ManifestDigest::new([0; 32]), + config: None, + config_schema: None, + program: None, + slots: BTreeMap::new(), + provides: BTreeMap::new(), + exports: BTreeMap::new(), + resources: BTreeMap::new(), + child_templates: BTreeMap::new(), + metadata: None, + }, + ComponentIr { + id: consumer_id, + moniker: "/job-1/consumer".to_string(), + parent: Some(child_root_id), + children: Vec::new(), + resolved_url: Some("file:///tmp/job-1-consumer.json5".to_string()), + digest: amber_manifest::ManifestDigest::new([0; 32]), + config: None, + config_schema: None, + program: None, + slots: BTreeMap::new(), + provides: BTreeMap::new(), + exports: BTreeMap::new(), + resources: BTreeMap::new(), + child_templates: BTreeMap::new(), + metadata: None, + }, + ], + bindings: Vec::new(), + }), + input_bindings: Vec::new(), + assignments: BTreeMap::from([ + ("/job-1".to_string(), "direct_local".to_string()), + ("/job-1/provider".to_string(), "direct_local".to_string()), + ("/job-1/consumer".to_string(), "compose_local".to_string()), + ]), + overlay_ids: vec![ + "child:1:consumer:compose_local:amber_link_test".to_string(), + "child:1:provider:direct_local:amber_export_test".to_string(), + ], + overlays: vec![ + DynamicOverlayRecord { + overlay_id: "child:1:consumer:compose_local:amber_link_test".to_string(), + site_id: "compose_local".to_string(), + action: DynamicOverlayAction::ExternalSlot { link: link.clone() }, + }, + DynamicOverlayRecord { + overlay_id: "child:1:provider:direct_local:amber_export_test".to_string(), + site_id: "direct_local".to_string(), + action: DynamicOverlayAction::ExportPeer { link: link.clone() }, + }, + ], + outputs: BTreeMap::new(), + }; + state.pending_creates.push(pending_create(1, child.clone())); + write_control_state(&state_path, &state).expect("state should write"); + + let peer_router = Router::new() + .route( + "/v1/controller/router-identity", + axum::routing::get(|| async { + Json(RouterIdentityResponse { + id: "/site/compose_local/router".to_string(), + public_key_b64: base64::engine::general_purpose::STANDARD.encode([13u8; 32]), + }) + }), + ) + .route( + "/v1/internal/children/prepare", + axum::routing::post(|| async { StatusCode::NO_CONTENT }), + ) + .route( + "/v1/internal/children/publish", + axum::routing::post(|| async { StatusCode::NO_CONTENT }), + ) + .route( + "/v1/internal/children/destroy", + axum::routing::post(|| async { StatusCode::NO_CONTENT }), + ) + .route( + "/v1/internal/children/rollback", + axum::routing::post(|| async { StatusCode::NO_CONTENT }), + ) + .route( + "/v1/internal/link-overlays/external-slot/publish", + axum::routing::post(|| async { StatusCode::NO_CONTENT }), + ) + .route( + "/v1/internal/link-overlays/external-slot/clear", + axum::routing::post(|| async { StatusCode::NO_CONTENT }), + ); + let (peer_router_url, _peer_router_handle) = spawn_test_router(peer_router).await; + + let publish_calls = Arc::new(std::sync::Mutex::new(Vec::new())); + let resolve_calls = Arc::new(std::sync::Mutex::new(Vec::new())); + let runtime = Arc::new(PublishBeforeResolveRuntime { + publish_calls: publish_calls.clone(), + resolve_calls: resolve_calls.clone(), + }); + let base_app = test_control_state_app(&dir, state, state_path); + let mut plan = base_app.controller_plan.as_ref().clone(); + plan.site_id = "direct_local".to_string(); + plan.kind = SiteKind::Direct; + plan.router_identity_id = "/site/direct_local/router".to_string(); + plan.peer_site_router_urls = BTreeMap::from([("compose_local".to_string(), peer_router_url)]); + plan.site_state_root = dir + .path() + .join("state") + .join("direct_local") + .display() + .to_string(); + plan.artifact_dir = dir + .path() + .join("artifact") + .join("direct_local") + .display() + .to_string(); + plan.storage_root = Some( + dir.path() + .join("storage") + .join("direct_local") + .display() + .to_string(), + ); + plan.runtime_root = Some( + dir.path() + .join("runtime") + .join("direct_local") + .display() + .to_string(), + ); + let app = ControlStateApp { + controller_plan: Arc::new(plan), + runtime, + ..base_app + }; + continue_create_committed_hidden(&app, child.child_id) + .await + .expect("create should publish the child before resolving cross-site overlays"); + + assert!( + !publish_calls + .lock() + .expect("publish call log mutex should lock") + .is_empty(), + "local child publication should happen before overlay resolution", + ); + assert!( + !resolve_calls + .lock() + .expect("resolve call log mutex should lock") + .is_empty(), + "cross-site overlay resolution should still occur for the compose consumer", + ); + let live_state = app.control_state.lock().await.clone(); + assert!( + live_state + .live_children + .iter() + .any(|live_child| live_child.child_id == child.child_id), + "the committed-hidden child should become live after publication", + ); +} + #[tokio::test] async fn destroy_and_recreate_same_child_name_gets_a_new_capability_instance_id() { let dir = TempDir::new().expect("temp dir"); @@ -6844,7 +7268,7 @@ async fn prepare_child_record_uses_frozen_dynamic_placement_assignments() { } #[tokio::test] -async fn prepare_child_record_rejects_cross_site_dynamic_fragments() { +async fn prepare_child_record_supports_cross_site_dynamic_fragments() { let dir = TempDir::new().expect("temp dir"); let root_path = dir.path().join("root.json5"); let child_path = dir.path().join("child-compose.json5"); @@ -7052,7 +7476,7 @@ async fn prepare_child_record_rejects_cross_site_dynamic_fragments() { let mut state = compile_control_state_with_placement(&root_path, Some(&placement)).await; let root_authority = state.base_scenario.root; - let err = prepare_child_record( + let child = prepare_child_record( &mut state, root_authority, &CreateChildRequest { @@ -7064,12 +7488,35 @@ async fn prepare_child_record_rejects_cross_site_dynamic_fragments() { }, ) .await - .expect_err("site controllers must reject dynamic children that span multiple sites"); - assert_eq!(err.code, ProtocolErrorCode::PlacementUnsatisfied); - assert!( - err.message.contains("spans multiple sites"), - "cross-site dynamic child placement should be rejected explicitly, got: {}", - err.message + .expect("cross-site dynamic children should retain their multi-site placement"); + assert_eq!( + child_runtime_site_ids(&child).expect("child site ids"), + vec![ + "compose_local".to_string(), + "direct_local".to_string(), + "kind_local".to_string(), + "vm_local".to_string() + ] + ); + let compose_spec = + build_local_child_runtime_spec(&state, &child, "compose_local").expect("compose spec"); + assert_eq!(compose_spec.assigned_components, vec!["/job-compose/root"]); + let direct_spec = + build_local_child_runtime_spec(&state, &child, "direct_local").expect("direct spec"); + assert_eq!( + direct_spec.assigned_components, + vec!["/job-compose/direct_helper"] + ); + let kind_spec = + build_local_child_runtime_spec(&state, &child, "kind_local").expect("kind spec"); + assert_eq!( + kind_spec.assigned_components, + vec!["/job-compose/kind_helper"] + ); + let vm_spec = build_local_child_runtime_spec(&state, &child, "vm_local").expect("vm spec"); + assert_eq!( + vm_spec.assigned_components, + vec!["/job-compose/vm_helper/root"] ); } @@ -8613,12 +9060,33 @@ fn kubernetes_site_controller_resources_are_injected_into_the_artifact() { let site_state_root = temp.path().join("state").join("kube-site"); fs::create_dir_all(artifact_root.join("05-networkpolicies")) .expect("network policies dir should create"); + fs::create_dir_all(artifact_root.join("04-services")).expect("services dir should create"); fs::create_dir_all(site_state_root.clone()).expect("site state root should create"); fs::write( artifact_root.join("kustomization.yaml"), "resources:\n - 05-networkpolicies/amber-router-netpol.yaml\n", ) .expect("kustomization should write"); + fs::write( + artifact_root.join("04-services/amber-router.yaml"), + r#" +apiVersion: v1 +kind: Service +metadata: + name: amber-router +spec: + ports: + - name: mesh + port: 24000 + targetPort: 24000 + protocol: TCP + - name: control + port: 24100 + targetPort: 24100 + protocol: TCP +"#, + ) + .expect("router service should write"); fs::write( artifact_root.join("05-networkpolicies/amber-router-netpol.yaml"), r#" @@ -8641,6 +9109,10 @@ spec: "{\"schema\":\"amber.test\",\"version\":1}", ) .expect("desired links should write"); + let peer_site_router_urls = BTreeMap::from([( + "peer-site".to_string(), + "http://amber-router:37046".to_string(), + )]); let plan = write_site_controller_plan( &site_controller_plan_path(&site_state_root), @@ -8651,7 +9123,7 @@ spec: SocketAddr::from(([0, 0, 0, 0], SITE_CONTROLLER_PORT)), &format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{SITE_CONTROLLER_PORT}"), "/site/kube-site/router", - &BTreeMap::new(), + &peer_site_router_urls, &BTreeMap::new(), &BTreeMap::new(), Some("amber-router:24100"), @@ -8687,12 +9159,31 @@ spec: assert!(kustomization.contains("03-deployments/amber-site-controller.yaml")); assert!(kustomization.contains("04-services/amber-site-controller.yaml")); - let deployment = - fs::read_to_string(artifact_root.join("03-deployments/amber-site-controller.yaml")) - .expect("deployment should read"); - assert!(deployment.contains("amber-site-controller")); - assert!(deployment.contains("ghcr.io/rdi-foundation/amber-site-controller:test")); - assert!(deployment.contains("/amber/site/state/site-controller-plan.json")); + let deployment_path = artifact_root.join("03-deployments/amber-site-controller.yaml"); + let deployment_raw = fs::read_to_string(&deployment_path).expect("deployment should read"); + assert!(deployment_raw.contains("amber-site-controller")); + assert!(deployment_raw.contains("ghcr.io/rdi-foundation/amber-site-controller:test")); + assert!(deployment_raw.contains("/amber/site/state/site-controller-plan.json")); + let deployment: serde_yaml::Value = + serde_yaml::from_str(&deployment_raw).expect("deployment should parse"); + let container = deployment["spec"]["template"]["spec"]["containers"] + .as_sequence() + .and_then(|containers| containers.first()) + .and_then(serde_yaml::Value::as_mapping) + .expect("deployment should contain a site-controller container"); + assert!( + !container.contains_key(serde_yaml::Value::String("command".to_string())), + "kubernetes site controller should keep the image entrypoint and pass only args" + ); + let args = container + .get(serde_yaml::Value::String("args".to_string())) + .and_then(serde_yaml::Value::as_sequence) + .expect("deployment should pass site-controller args"); + assert_eq!(args[0].as_str(), Some("--plan")); + assert_eq!( + args[1].as_str(), + Some("/amber/site/state/site-controller-plan.json") + ); let seed = fs::read_to_string(artifact_root.join("01-configmaps/amber-site-controller-seed.yaml")) @@ -8706,6 +9197,11 @@ spec: .expect("router netpol should read"); assert!(router_netpol.contains("amber-site-controller")); assert!(router_netpol.contains("24100")); + assert!(router_netpol.contains("37046")); + let router_service = fs::read_to_string(artifact_root.join("04-services/amber-router.yaml")) + .expect("router service should read"); + assert!(router_service.contains("controller-route-37046")); + assert!(router_service.contains("port: 37046")); } #[test] From 95555a10efad11c3430c43745f8036fa67da258e Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Wed, 15 Apr 2026 18:05:02 +0800 Subject: [PATCH 06/25] Fix some more tests --- cli/src/direct_runtime/process.rs | 73 +- cli/src/main.rs | 1 + cli/src/mixed_run.rs | 240 ++---- cli/src/mixed_run/launch_bundle.rs | 58 +- cli/src/mixed_run/outside_proxy.rs | 152 +++- cli/src/mixed_run/supervisor.rs | 431 ++++++---- cli/src/mixed_run/tests.rs | 68 ++ cli/src/tests.rs | 71 ++ cli/src/unix_process.rs | 185 ++++ cli/tests/direct_smoke.rs | 9 +- cli/tests/linux_vm.rs | 252 +----- cli/tests/mixed_run/smoke.rs | 146 ++-- cli/tests/mixed_run/support.rs | 805 +++++++++++++++--- cli/tests/test_support/port_allocator.rs | 63 ++ cli/tests/vm_smoke.rs | 8 +- .../src/targets/mesh/docker_compose/mod.rs | 3 + .../mesh/docker_compose/tests/rendering.rs | 7 + runtime/proxy/src/lib.rs | 39 +- runtime/proxy/src/tests.rs | 42 + runtime/router/src/dynamic_caps.rs | 178 ++-- runtime/router/src/tests.rs | 209 +++++ .../site-controller/src/control_state_api.rs | 85 +- .../site-controller/src/default_runtime.rs | 349 ++++---- .../src/default_runtime/child_runtime.rs | 245 ++++-- .../src/default_runtime/compose_controller.rs | 149 +++- .../src/default_runtime/dynamic_routes.rs | 5 +- .../default_runtime/kubernetes_controller.rs | 129 ++- .../src/default_runtime/site_artifacts.rs | 65 +- .../default_runtime/site_runtime_support.rs | 638 +++++++++----- runtime/site-controller/src/orchestration.rs | 150 ++-- runtime/site-controller/src/planner.rs | 4 + runtime/site-controller/src/runtime_api.rs | 27 +- .../site-controller/src/site_controller.rs | 18 +- runtime/site-controller/src/state.rs | 9 + runtime/site-controller/src/tests.rs | 485 +++++++++-- 35 files changed, 3894 insertions(+), 1504 deletions(-) create mode 100644 cli/src/unix_process.rs create mode 100644 cli/tests/test_support/port_allocator.rs diff --git a/cli/src/direct_runtime/process.rs b/cli/src/direct_runtime/process.rs index dd31ffa5..1671930d 100644 --- a/cli/src/direct_runtime/process.rs +++ b/cli/src/direct_runtime/process.rs @@ -636,23 +636,30 @@ pub(crate) async fn supervise_children( } } +#[cfg(all(unix, not(target_os = "linux")))] pub(crate) async fn terminate_children(children: &mut [ManagedChild]) { + let wrapper_pids = children + .iter() + .filter_map(|child| child.wrapper.as_ref().and_then(tokio::process::Child::id)) + .collect::>(); + let _ = + crate::unix_process::terminate_process_roots(&wrapper_pids, DIRECT_SHUTDOWN_GRACE_PERIOD) + .await; for child in children.iter_mut() { - #[cfg(target_os = "linux")] - { - if linux_pid_is_alive(child.managed_pid) { - let _ = send_sigterm(child.managed_pid); - } - if child.wrapper_pid != child.managed_pid { - let _ = send_sigterm(child.wrapper_pid); - } + if let Some(mut wrapper) = child.wrapper.take() { + let _ = wrapper.wait().await; } - #[cfg(not(target_os = "linux"))] - if let Some(wrapper) = child.wrapper.as_mut() - && wrapper.try_wait().ok().flatten().is_none() - && let Some(pid) = wrapper.id() - { - let _ = send_sigterm(pid); + } +} + +#[cfg(target_os = "linux")] +pub(crate) async fn terminate_children(children: &mut [ManagedChild]) { + for child in children.iter_mut() { + if linux_pid_is_alive(child.managed_pid) { + let _ = send_sigterm(child.managed_pid); + } + if child.wrapper_pid != child.managed_pid { + let _ = send_sigterm(child.wrapper_pid); } } @@ -660,28 +667,17 @@ pub(crate) async fn terminate_children(children: &mut [ManagedChild]) { loop { let mut all_exited = true; for child in children.iter_mut() { - #[cfg(target_os = "linux")] if let Some(wrapper) = child.wrapper.as_mut() && wrapper.try_wait().ok().flatten().is_some() { child.wrapper = None; } - #[cfg(target_os = "linux")] if linux_pid_is_alive(child.managed_pid) { all_exited = false; } - #[cfg(target_os = "linux")] if child.wrapper.is_some() { all_exited = false; } - #[cfg(not(target_os = "linux"))] - if child - .wrapper - .as_mut() - .is_some_and(|wrapper| wrapper.try_wait().ok().flatten().is_none()) - { - all_exited = false; - } } if all_exited || Instant::now() >= deadline { break; @@ -690,16 +686,23 @@ pub(crate) async fn terminate_children(children: &mut [ManagedChild]) { } for child in children.iter_mut() { - #[cfg(target_os = "linux")] - { - if linux_pid_is_alive(child.managed_pid) { - let _ = kill_pid_force(child.managed_pid); - } - if child.wrapper_pid != child.managed_pid { - let _ = kill_pid_force(child.wrapper_pid); - } + if linux_pid_is_alive(child.managed_pid) { + let _ = kill_pid_force(child.managed_pid); + } + if child.wrapper_pid != child.managed_pid { + let _ = kill_pid_force(child.wrapper_pid); + } + } + for child in children.iter_mut() { + if let Some(mut wrapper) = child.wrapper.take() { + let _ = wrapper.wait().await; } - #[cfg(not(target_os = "linux"))] + } +} + +#[cfg(not(unix))] +pub(crate) async fn terminate_children(children: &mut [ManagedChild]) { + for child in children.iter_mut() { if let Some(wrapper) = child.wrapper.as_mut() && wrapper.try_wait().ok().flatten().is_none() { @@ -713,7 +716,7 @@ pub(crate) async fn terminate_children(children: &mut [ManagedChild]) { } } -#[cfg(unix)] +#[cfg(target_os = "linux")] pub(crate) fn send_sigterm(pid: u32) -> std::result::Result<(), ()> { let pid = i32::try_from(pid).map_err(|_| ())?; let rc = unsafe { libc::kill(pid, libc::SIGTERM) }; diff --git a/cli/src/main.rs b/cli/src/main.rs index 32aacc49..007b05d5 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -6,6 +6,7 @@ mod run_inputs; mod run_logs; mod site_proxy_metadata; mod tcp_readiness; +mod unix_process; mod vm_runtime; #[cfg(target_os = "linux")] diff --git a/cli/src/mixed_run.rs b/cli/src/mixed_run.rs index 784ee0fb..568d5f79 100644 --- a/cli/src/mixed_run.rs +++ b/cli/src/mixed_run.rs @@ -1,5 +1,5 @@ use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, + collections::{BTreeMap, BTreeSet}, env, fs, future::Future, io::{Read as _, Write as _}, @@ -46,6 +46,7 @@ use crate::{ run_inputs::{collect_run_interface, validate_export_bindings, validate_slot_bindings}, site_proxy_metadata::load_site_proxy_metadata, tcp_readiness::{wait_for_http_response, wait_for_stable_endpoint}, + unix_process::{pid_is_alive, terminate_process_roots}, vm_runtime::{ TCG_VM_STARTUP_TIMEOUT, VmLaunchPreview, VmRuntimeState, VmSiteLaunchPreview, build_vm_site_launch_preview, vm_current_control_socket_path, vm_uses_tcg_accel, @@ -206,18 +207,9 @@ pub(crate) struct RunReceipt { pub(crate) run_root: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) observability: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub(crate) bridge_proxies: Vec, pub(crate) sites: BTreeMap, } -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct BridgeProxyReceipt { - pub(crate) export_name: String, - pub(crate) pid: u32, - pub(crate) listen: String, -} - #[derive(Clone, Debug, Serialize, Deserialize)] struct LaunchBundleManifest { schema: String, @@ -362,6 +354,10 @@ pub(crate) struct SiteReceipt { #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) router_mesh_addr: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) compose_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) kubernetes_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) router_identity_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) router_public_key_b64: Option, @@ -406,6 +402,10 @@ struct SiteManagerState { #[serde(default, skip_serializing_if = "Option::is_none")] router_mesh_addr: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + compose_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + kubernetes_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] router_identity_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] router_public_key_b64: Option, @@ -634,19 +634,6 @@ struct SupervisorChild { child: Child, } -pub(crate) struct BridgeProxyHandle { - pub(crate) child: Child, - pub(crate) export_name: String, - pub(crate) listen: SocketAddr, -} - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub(crate) struct BridgeProxyKey { - pub(crate) provider_output_dir: String, - pub(crate) export_name: String, - pub(crate) consumer_kind: SiteKind, -} - #[derive(Debug)] struct SupervisorRuntime { site_process: Option, @@ -691,6 +678,8 @@ fn site_state_from_receipt( context: site.context.clone(), router_control: site.router_control.clone(), router_mesh_addr: site.router_mesh_addr.clone(), + compose_consumer_router_mesh_addr: site.compose_consumer_router_mesh_addr.clone(), + kubernetes_consumer_router_mesh_addr: site.kubernetes_consumer_router_mesh_addr.clone(), router_identity_id: site.router_identity_id.clone(), router_public_key_b64: site.router_public_key_b64.clone(), site_controller_pid: site.site_controller_pid, @@ -699,6 +688,28 @@ fn site_state_from_receipt( } } +fn published_container_consumer_router_mesh_addrs( + site_kind: SiteKind, + router_mesh_addr: Option<&str>, +) -> (Option, Option) { + let Some(router_mesh_addr) = router_mesh_addr else { + return (None, None); + }; + let compose = amber_site_controller::router_mesh_addr_for_consumer( + site_kind, + SiteKind::Compose, + router_mesh_addr, + ) + .unwrap_or_else(|err| panic!("failed to compute compose consumer mesh addr: {err}")); + let kubernetes = amber_site_controller::router_mesh_addr_for_consumer( + site_kind, + SiteKind::Kubernetes, + router_mesh_addr, + ) + .unwrap_or_else(|err| panic!("failed to compute kubernetes consumer mesh addr: {err}")); + (Some(compose), Some(kubernetes)) +} + enum SiteSupervisorStopStatus { Graceful { shutdown_failed: bool }, Exited, @@ -832,6 +843,8 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { format!("stopped site supervisor {}", plan.site_id), &[("amber.run_id", plan.run_id.clone())], ); + let (compose_consumer_router_mesh_addr, kubernetes_consumer_router_mesh_addr) = + published_container_consumer_router_mesh_addrs(plan.kind, None); write_site_state_if_changed( &state_path, &mut last_written_state, @@ -851,6 +864,8 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { context: plan.context.clone(), router_control: None, router_mesh_addr: None, + compose_consumer_router_mesh_addr, + kubernetes_consumer_router_mesh_addr, router_identity_id: None, router_public_key_b64: None, site_controller_pid: None, @@ -865,6 +880,8 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { && coordinator_has_exited(&run_root, plan.coordinator_pid)? { cleanup_site(&plan, &mut runtime).await?; + let (compose_consumer_router_mesh_addr, kubernetes_consumer_router_mesh_addr) = + published_container_consumer_router_mesh_addrs(plan.kind, None); write_site_state_if_changed( &state_path, &mut last_written_state, @@ -884,6 +901,8 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { context: plan.context.clone(), router_control: None, router_mesh_addr: None, + compose_consumer_router_mesh_addr, + kubernetes_consumer_router_mesh_addr, router_identity_id: None, router_public_key_b64: None, site_controller_pid: None, @@ -973,6 +992,12 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { let public_key_b64 = base64::engine::general_purpose::STANDARD .encode(discovery.router_identity.public_key); + let router_mesh_addr = discovery.router_addr.map(|addr| addr.to_string()); + let (compose_consumer_router_mesh_addr, kubernetes_consumer_router_mesh_addr) = + published_container_consumer_router_mesh_addrs( + plan.kind, + router_mesh_addr.as_deref(), + ); write_site_state_if_changed( &state_path, &mut last_written_state, @@ -991,7 +1016,9 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { port_forward_pid: runtime.port_forward.as_ref().map(Child::id), context: plan.context.clone(), router_control: Some(discovery.control_endpoint.to_string()), - router_mesh_addr: discovery.router_addr.map(|addr| addr.to_string()), + router_mesh_addr, + compose_consumer_router_mesh_addr, + kubernetes_consumer_router_mesh_addr, router_identity_id: Some(discovery.router_identity.id), router_public_key_b64: Some(public_key_b64), site_controller_pid: None, @@ -1400,23 +1427,6 @@ pub(crate) fn decode_public_key(value: &str) -> Result<[u8; 32]> { .map_err(|_| miette::miette!("invalid router public key length")) } -fn pid_is_alive(pid: u32) -> bool { - #[cfg(unix)] - { - let alive = unsafe { - libc::kill(pid as i32, 0) == 0 - || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) - }; - alive && process_status_code(pid) != Some('Z') - } - - #[cfg(not(unix))] - { - let _ = pid; - true - } -} - fn send_sigterm(pid: u32) { #[cfg(unix)] { @@ -1572,160 +1582,38 @@ fn handle_otlp_connection(stream: &mut TcpStream, requests_log: &Path) -> Result Ok(()) } -#[cfg(unix)] -fn send_sigkill(pid: u32) { - let _ = unsafe { libc::kill(pid as i32, libc::SIGKILL) }; -} - #[cfg(unix)] async fn shutdown_recorded_processes(site: &SiteReceipt) -> Result<()> { - let mut roots = Vec::new(); - if let Some(pid) = site.process_pid { - roots.push(pid); - } - if let Some(pid) = site.port_forward_pid { - roots.push(pid); - } - terminate_recorded_processes(&roots).await + terminate_recorded_processes(&recorded_process_roots(site)).await } #[cfg(not(unix))] async fn shutdown_recorded_processes(site: &SiteReceipt) -> Result<()> { - if let Some(pid) = site.process_pid { - send_sigterm(pid); - } - if let Some(pid) = site.port_forward_pid { + for pid in recorded_process_roots(site) { send_sigterm(pid); } Ok(()) } -#[cfg(unix)] -async fn terminate_recorded_processes(root_pids: &[u32]) -> Result<()> { - let mut seen = BTreeSet::new(); - let mut ordered = Vec::new(); - for root_pid in root_pids { - for pid in process_tree_postorder(*root_pid)? { - if seen.insert(pid) { - ordered.push(pid); - } - } - } - if ordered.is_empty() { - return Ok(()); - } - for pid in &ordered { - send_sigterm(*pid); - } - wait_for_pids_exit(&ordered, PROCESS_SHUTDOWN_GRACE_PERIOD).await; - let survivors = ordered - .iter() - .copied() - .filter(|pid| pid_is_alive(*pid)) - .collect::>(); - for pid in &survivors { - send_sigkill(*pid); - } - wait_for_pids_exit(&survivors, Duration::from_secs(2)).await; - Ok(()) -} - -#[cfg(unix)] -fn process_status_code(pid: u32) -> Option { - let output = Command::new("ps") - .arg("-o") - .arg("stat=") - .arg("-p") - .arg(pid.to_string()) - .output() - .ok()?; - if !output.status.success() { - return None; +fn recorded_process_roots(site: &SiteReceipt) -> Vec { + let mut roots = vec![site.supervisor_pid]; + if let Some(pid) = site.process_pid { + roots.push(pid); } - parse_process_status_code(&String::from_utf8_lossy(&output.stdout)) -} - -#[cfg(unix)] -fn parse_process_status_code(raw: &str) -> Option { - raw.split_whitespace() - .next()? - .chars() - .next() - .map(|state| state.to_ascii_uppercase()) -} - -#[cfg(unix)] -async fn wait_for_pids_exit(pids: &[u32], timeout: Duration) { - let deadline = Instant::now() + timeout; - while Instant::now() < deadline { - if pids.iter().all(|pid| !pid_is_alive(*pid)) { - return; - } - sleep(Duration::from_millis(100)).await; + if let Some(pid) = site.port_forward_pid { + roots.push(pid); } + roots } #[cfg(unix)] -fn process_tree_postorder(root_pid: u32) -> Result> { - let output = Command::new("ps") - .arg("-axo") - .arg("pid=,ppid=") - .output() - .into_diagnostic() - .wrap_err("failed to enumerate process tree")?; - if !output.status.success() { - return Err(miette::miette!( - "failed to enumerate process tree: status {}", - output.status - )); - } - - let parent_by_pid = parse_process_table(&String::from_utf8_lossy(&output.stdout))?; - let mut children_by_parent = HashMap::>::new(); - for (pid, ppid) in parent_by_pid { - children_by_parent.entry(ppid).or_default().push(pid); - } - - let mut ordered = Vec::new(); - collect_process_tree_postorder(root_pid, &children_by_parent, &mut ordered); - Ok(ordered) +async fn terminate_recorded_processes(root_pids: &[u32]) -> Result<()> { + terminate_process_roots(root_pids, PROCESS_SHUTDOWN_GRACE_PERIOD).await } #[cfg(unix)] -fn collect_process_tree_postorder( - pid: u32, - children_by_parent: &HashMap>, - ordered: &mut Vec, -) { - if let Some(children) = children_by_parent.get(&pid) { - for child in children { - collect_process_tree_postorder(*child, children_by_parent, ordered); - } - } - ordered.push(pid); -} - -fn parse_process_table(raw: &str) -> Result> { - let mut parent_by_pid = HashMap::new(); - for line in raw.lines() { - let mut fields = line.split_whitespace(); - let Some(pid) = fields.next() else { - continue; - }; - let Some(ppid) = fields.next() else { - continue; - }; - let pid = pid - .parse::() - .into_diagnostic() - .wrap_err_with(|| format!("invalid process table pid `{pid}`"))?; - let ppid = ppid - .parse::() - .into_diagnostic() - .wrap_err_with(|| format!("invalid process table parent pid `{ppid}`"))?; - parent_by_pid.insert(pid, ppid); - } - Ok(parent_by_pid) +fn send_sigkill(pid: u32) { + let _ = unsafe { libc::kill(pid as i32, libc::SIGKILL) }; } fn find_header_end(buf: &[u8]) -> Option { diff --git a/cli/src/mixed_run/launch_bundle.rs b/cli/src/mixed_run/launch_bundle.rs index 1dd62e2c..d4ee7ace 100644 --- a/cli/src/mixed_run/launch_bundle.rs +++ b/cli/src/mixed_run/launch_bundle.rs @@ -301,6 +301,25 @@ pub(super) fn materialize_launch_bundle( )?; let local_router_control = site_controller_local_router_control(site.site_plan.site.kind, &site.artifact_dir); + let published_router_mesh_addr = planned_router_mesh_addrs.get(site_id).map(String::as_str); + let compose_consumer_router_mesh_addr = published_router_mesh_addr + .map(|addr| { + amber_site_controller::router_mesh_addr_for_consumer( + site.site_plan.site.kind, + SiteKind::Compose, + addr, + ) + }) + .transpose()?; + let kubernetes_consumer_router_mesh_addr = published_router_mesh_addr + .map(|addr| { + amber_site_controller::router_mesh_addr_for_consumer( + site.site_plan.site.kind, + SiteKind::Kubernetes, + addr, + ) + }) + .transpose()?; amber_site_controller::write_site_controller_plan( &site.controller_plan_path, @@ -315,7 +334,9 @@ pub(super) fn materialize_launch_bundle( &peer_router_identities, &peer_router_mesh_addrs, Some(local_router_control.as_str()), - planned_router_mesh_addrs.get(site_id).map(String::as_str), + published_router_mesh_addr, + compose_consumer_router_mesh_addr.as_deref(), + kubernetes_consumer_router_mesh_addr.as_deref(), &site.controller_state_path, bundle_root, &state_root, @@ -1081,7 +1102,6 @@ pub(crate) async fn run_run_plan_with_id( let mut launched_by_site = BTreeMap::::new(); let mut started_site_receipts = BTreeMap::::new(); let mut supervisor_children = BTreeMap::::new(); - let mut bridge_proxies = BTreeMap::::new(); let test_wave_delay = test_wave_delay()?; let result = async { @@ -1120,7 +1140,6 @@ pub(crate) async fn run_run_plan_with_id( &launched_by_site, &run_root, &state_root, - &mut bridge_proxies, ) .await?; @@ -1154,14 +1173,6 @@ pub(crate) async fn run_run_plan_with_id( source_plan_path: source_plan_path.map(|path| path.display().to_string()), run_root: run_root.display().to_string(), observability: observability_receipt.clone(), - bridge_proxies: bridge_proxies - .values() - .map(|proxy| BridgeProxyReceipt { - export_name: proxy.export_name.clone(), - pid: proxy.child.id(), - listen: proxy.listen.to_string(), - }) - .collect(), sites: launched_by_site .into_iter() .map(|(site_id, launched)| (site_id, launched.receipt)) @@ -1174,15 +1185,9 @@ pub(crate) async fn run_run_plan_with_id( if result.is_err() { let _ = write_stop_marker(&run_root); - for bridge in bridge_proxies.values_mut() { - send_sigterm(bridge.child.id()); - } for supervisor in supervisor_children.values_mut() { send_sigterm(supervisor.child.id()); } - for bridge in bridge_proxies.values_mut() { - let _ = wait_for_child_exit(&mut bridge.child, PROCESS_SHUTDOWN_GRACE_PERIOD).await; - } for supervisor in supervisor_children.values_mut() { let _ = wait_for_child_exit(&mut supervisor.child, PROCESS_SHUTDOWN_GRACE_PERIOD).await; } @@ -1218,6 +1223,12 @@ pub(crate) async fn run_run_plan_with_id( context: receipt.context.clone(), router_control: receipt.router_control.clone(), router_mesh_addr: receipt.router_mesh_addr.clone(), + compose_consumer_router_mesh_addr: receipt + .compose_consumer_router_mesh_addr + .clone(), + kubernetes_consumer_router_mesh_addr: receipt + .kubernetes_consumer_router_mesh_addr + .clone(), router_identity_id: receipt.router_identity_id.clone(), router_public_key_b64: receipt.router_public_key_b64.clone(), site_controller_pid: receipt.site_controller_pid, @@ -1341,19 +1352,6 @@ pub(crate) async fn stop_run(run_id: &str, storage_root_override: Option<&Path>) PROCESS_SHUTDOWN_GRACE_PERIOD.as_secs() )); } - for proxy in &receipt.bridge_proxies { - send_sigterm(proxy.pid); - } - for proxy in &receipt.bridge_proxies { - if !wait_for_pid_exit(proxy.pid, PROCESS_SHUTDOWN_GRACE_PERIOD).await { - shutdown_failures.push(format!( - "bridge proxy `{}` (pid {}) did not stop within {}s", - proxy.export_name, - proxy.pid, - PROCESS_SHUTDOWN_GRACE_PERIOD.as_secs() - )); - } - } if !shutdown_failures.is_empty() { return Err(miette::miette!( diff --git a/cli/src/mixed_run/outside_proxy.rs b/cli/src/mixed_run/outside_proxy.rs index b7ab1a42..d25168c7 100644 --- a/cli/src/mixed_run/outside_proxy.rs +++ b/cli/src/mixed_run/outside_proxy.rs @@ -344,22 +344,13 @@ pub(super) async fn stop_site_from_receipt( SiteKind::Compose => { if let Some(project_name) = site.compose_project.as_deref() { let launch_env = read_compose_launch_env(run_root, site_id)?; - let status = compose_command(Some(project_name), Path::new(&site.artifact_dir)) - .envs(launch_env) - .arg("down") - .arg("-v") - .arg("--remove-orphans") - .status() - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to stop orphaned compose site `{project_name}`") - })?; - if !status.success() { - return Err(miette::miette!( - "orphaned compose site `{project_name}` failed to stop with status \ - {status}" - )); - } + stop_compose_site_with_docker( + Path::new("docker"), + project_name, + Path::new(&site.artifact_dir), + &launch_env, + Duration::from_secs(30), + )?; } } SiteKind::Kubernetes => { @@ -390,6 +381,74 @@ pub(super) async fn stop_site_from_receipt( Ok(()) } +fn stop_compose_site_with_docker( + docker_bin: &Path, + project_name: &str, + artifact_dir: &Path, + launch_env: &BTreeMap, + timeout: Duration, +) -> Result<()> { + let status = compose_command_with_docker_bin(docker_bin, Some(project_name), artifact_dir) + .envs(launch_env) + .arg("down") + .arg("-v") + .arg("--remove-orphans") + .status() + .into_diagnostic() + .wrap_err_with(|| format!("failed to stop orphaned compose site `{project_name}`"))?; + if !status.success() { + return Err(miette::miette!( + "orphaned compose site `{project_name}` failed to stop with status {status}" + )); + } + + let deadline = Instant::now() + timeout; + loop { + let output = compose_command_with_docker_bin(docker_bin, Some(project_name), artifact_dir) + .envs(launch_env) + .arg("ps") + .arg("-q") + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to verify orphaned compose site `{project_name}` is stopped") + })?; + if !output.status.success() { + return Err(miette::miette!( + "failed to verify orphaned compose site `{project_name}` is \ + stopped\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + )); + } + if String::from_utf8_lossy(&output.stdout).trim().is_empty() { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(miette::miette!( + "timed out waiting for orphaned compose site `{project_name}` to fully stop" + )); + } + std::thread::sleep(Duration::from_millis(200)); + } +} + +fn compose_command_with_docker_bin( + docker_bin: &Path, + project_name: Option<&str>, + artifact_dir: &Path, +) -> Command { + let mut cmd = Command::new(docker_bin); + cmd.arg("compose") + .arg("-f") + .arg(artifact_dir.join("compose.yaml")); + if let Some(project_name) = project_name { + cmd.arg("-p").arg(project_name); + } + cmd.current_dir(artifact_dir); + cmd +} + pub(super) fn resolve_proxy_run_root( run_root: &Path, site_id: Option<&str>, @@ -641,3 +700,64 @@ pub(super) async fn wait_for_socket_listener(addr: SocketAddr) -> Result<()> { } Err(miette::miette!("timed out waiting for listener {}", addr)) } + +#[cfg(test)] +mod tests { + use std::os::unix::fs::PermissionsExt as _; + + use super::*; + + #[test] + fn stop_compose_site_waits_for_ps_to_empty_after_down_returns() { + let temp = tempfile::tempdir().expect("temp dir"); + let artifact_dir = temp.path().join("artifact"); + fs::create_dir_all(&artifact_dir).expect("artifact dir"); + fs::write(artifact_dir.join("compose.yaml"), "services: {}\n").expect("compose file"); + + let docker = temp.path().join("docker"); + let log_path = temp.path().join("docker.log"); + let count_path = temp.path().join("ps-count"); + fs::write( + &docker, + format!( + "#!/bin/sh\nset -eu\nlog_path='{}'\ncount_path='{}'\nprintf '%s\\n' \"$*\" >> \ + \"$log_path\"\nif [ \"${{1:-}}\" = compose ] && [ \"${{6:-}}\" = down ]; \ + then\nexit 0\nfi\nif [ \"${{1:-}}\" = compose ] && [ \"${{6:-}}\" = ps ] && [ \ + \"${{7:-}}\" = -q ]; then\ncount=0\nif [ -f \"$count_path\" ]; then\ncount=$(cat \ + \"$count_path\")\nfi\ncount=$((count + 1))\nprintf '%s' \"$count\" > \ + \"$count_path\"\nif [ \"$count\" -lt 3 ]; then\nprintf \ + 'container-id\\n'\nfi\nexit 0\nfi\necho \"unexpected docker invocation: $*\" \ + >&2\nexit 1\n", + log_path.display(), + count_path.display(), + ), + ) + .expect("docker stub"); + let mut perms = fs::metadata(&docker) + .expect("docker metadata") + .permissions(); + perms.set_mode(0o755); + fs::set_permissions(&docker, perms).expect("docker chmod"); + + stop_compose_site_with_docker( + &docker, + "amber-test-project", + &artifact_dir, + &BTreeMap::new(), + Duration::from_secs(1), + ) + .expect("compose stop should wait for project removal"); + + let log = fs::read_to_string(&log_path).expect("docker log"); + let ps_polls = log.lines().filter(|line| line.contains(" ps -q")).count(); + assert!( + log.lines() + .any(|line| line.contains(" down -v --remove-orphans")), + "expected compose down invocation in log:\n{log}" + ); + assert!( + ps_polls >= 3, + "expected repeated compose ps polling after down, got {ps_polls} polls:\n{log}" + ); + } +} diff --git a/cli/src/mixed_run/supervisor.rs b/cli/src/mixed_run/supervisor.rs index a788bfb4..c842c4c1 100644 --- a/cli/src/mixed_run/supervisor.rs +++ b/cli/src/mixed_run/supervisor.rs @@ -170,6 +170,12 @@ pub(super) fn launch_env( observability_endpoint: Option<&str>, ) -> Result> { let mut env = merge_env_maps(runtime_env, external_env); + if let Some(overrides) = std::env::var_os(amber_images::DEV_IMAGE_TAGS_ENV) + && let Some(overrides) = overrides.to_str() + { + env.entry(amber_images::DEV_IMAGE_TAGS_ENV.to_string()) + .or_insert_with(|| overrides.to_string()); + } env.insert(SCENARIO_RUN_ID_ENV.to_string(), run_id.to_string()); env.insert(SCENARIO_SCOPE_ENV.to_string(), mesh_scope.to_string()); if let Some(endpoint) = observability_endpoint { @@ -370,7 +376,6 @@ pub(super) async fn register_new_site_links( launched_by_site: &BTreeMap, run_root: &Path, state_root: &Path, - bridge_proxies: &mut BTreeMap, ) -> Result<()> { for link in links { if link.consumer_site == site_id { @@ -383,14 +388,8 @@ pub(super) async fn register_new_site_links( link.provider_site )); }; - let external_url = resolve_link_external_url( - provider, - link, - launched.receipt.kind, - run_root, - bridge_proxies, - ) - .await?; + let external_url = + resolve_link_external_url(provider, link, launched.receipt.kind, run_root).await?; let consumer_key = base64::engine::general_purpose::STANDARD .encode(launched.router_identity.public_key); @@ -441,14 +440,8 @@ pub(super) async fn register_new_site_links( let Some(consumer) = launched_by_site.get(&link.consumer_site) else { continue; }; - let external_url = resolve_link_external_url( - launched, - link, - consumer.receipt.kind, - run_root, - bridge_proxies, - ) - .await?; + let external_url = + resolve_link_external_url(launched, link, consumer.receipt.kind, run_root).await?; let consumer_key = base64::engine::general_purpose::STANDARD.encode(consumer.router_identity.public_key); @@ -583,6 +576,10 @@ pub(super) fn launched_site_from_state( context: state.context.clone(), router_control: state.router_control.clone(), router_mesh_addr: state.router_mesh_addr.clone(), + compose_consumer_router_mesh_addr: state.compose_consumer_router_mesh_addr.clone(), + kubernetes_consumer_router_mesh_addr: state + .kubernetes_consumer_router_mesh_addr + .clone(), router_identity_id: state.router_identity_id.clone(), router_public_key_b64: state.router_public_key_b64.clone(), site_controller_pid: state.site_controller_pid, @@ -1286,6 +1283,8 @@ pub(super) fn build_site_state( } else { (None, None, None, None) }; + let (compose_consumer_router_mesh_addr, kubernetes_consumer_router_mesh_addr) = + published_container_consumer_router_mesh_addrs(plan.kind, router_mesh_addr.as_deref()); SiteManagerState { schema: SITE_STATE_SCHEMA.to_string(), version: SITE_STATE_VERSION, @@ -1302,6 +1301,8 @@ pub(super) fn build_site_state( context: plan.context.clone(), router_control, router_mesh_addr, + compose_consumer_router_mesh_addr, + kubernetes_consumer_router_mesh_addr, router_identity_id, router_public_key_b64, site_controller_pid: runtime.site_controller.as_ref().map(Child::id), @@ -1340,6 +1341,14 @@ pub(super) fn persist_site_state( context: launched.receipt.context.clone(), router_control: launched.receipt.router_control.clone(), router_mesh_addr: launched.receipt.router_mesh_addr.clone(), + compose_consumer_router_mesh_addr: launched + .receipt + .compose_consumer_router_mesh_addr + .clone(), + kubernetes_consumer_router_mesh_addr: launched + .receipt + .kubernetes_consumer_router_mesh_addr + .clone(), router_identity_id: launched.receipt.router_identity_id.clone(), router_public_key_b64: launched.receipt.router_public_key_b64.clone(), site_controller_pid: launched.receipt.site_controller_pid, @@ -1349,6 +1358,39 @@ pub(super) fn persist_site_state( ) } +fn published_container_consumer_router_mesh_addrs( + provider_kind: SiteKind, + router_mesh_addr: Option<&str>, +) -> (Option, Option) { + let Some(router_mesh_addr) = router_mesh_addr else { + return (None, None); + }; + let compose = amber_site_controller::router_mesh_addr_for_consumer( + provider_kind, + SiteKind::Compose, + router_mesh_addr, + ) + .unwrap_or_else(|err| panic!("failed to compute compose consumer mesh addr: {err}")); + let kubernetes = amber_site_controller::router_mesh_addr_for_consumer( + provider_kind, + SiteKind::Kubernetes, + router_mesh_addr, + ) + .unwrap_or_else(|err| panic!("failed to compute kubernetes consumer mesh addr: {err}")); + (Some(compose), Some(kubernetes)) +} + +fn published_router_mesh_addr_for_consumer_kind( + receipt: &SiteReceipt, + consumer_kind: SiteKind, +) -> Option<&str> { + match consumer_kind { + SiteKind::Compose => receipt.compose_consumer_router_mesh_addr.as_deref(), + SiteKind::Kubernetes => receipt.kubernetes_consumer_router_mesh_addr.as_deref(), + SiteKind::Direct | SiteKind::Vm => None, + } +} + pub(super) fn write_site_state(path: &Path, state: SiteManagerState) -> Result<()> { write_json(path, &state) } @@ -1883,7 +1925,6 @@ pub(crate) async fn resolve_link_external_url( link: &RunLink, consumer_kind: SiteKind, run_root: &Path, - bridge_proxies: &mut BTreeMap, ) -> Result { resolve_link_external_url_for_output( provider, @@ -1891,7 +1932,6 @@ pub(crate) async fn resolve_link_external_url( link, consumer_kind, run_root, - bridge_proxies, ) .await } @@ -1901,130 +1941,16 @@ pub(crate) async fn resolve_link_external_url_for_output( provider_output_dir: &Path, link: &RunLink, consumer_kind: SiteKind, - run_root: &Path, - bridge_proxies: &mut BTreeMap, + _run_root: &Path, ) -> Result { - if !link_needs_bridge_proxy(provider.receipt.kind, consumer_kind) { - return external_slot_url(provider, provider_output_dir, link, consumer_kind); - } - - let port = ensure_bridge_proxy( - run_root, - provider, - provider_output_dir, - &link.export_name, - consumer_kind, - bridge_proxies, - ) - .await?; - bridge_proxy_external_url(port, link.protocol, consumer_kind) -} - -pub(super) fn link_needs_bridge_proxy(provider_kind: SiteKind, consumer_kind: SiteKind) -> bool { - matches!(consumer_kind, SiteKind::Compose | SiteKind::Kubernetes) - && provider_kind != SiteKind::Kubernetes -} - -pub(super) async fn ensure_bridge_proxy( - run_root: &Path, - provider: &LaunchedSite, - provider_output_dir: &Path, - export_name: &str, - consumer_kind: SiteKind, - bridge_proxies: &mut BTreeMap, -) -> Result { - let key = BridgeProxyKey { - provider_output_dir: provider_output_dir.display().to_string(), - export_name: export_name.to_string(), - consumer_kind, - }; - if let Some(proxy) = bridge_proxies.get_mut(&key) - && proxy.child.try_wait().into_diagnostic()?.is_none() - { - return Ok(proxy.listen.port()); - } - - let listen = bridge_proxy_bind_addr(consumer_kind, reserve_loopback_port()?); - let child = spawn_bridge_proxy(run_root, provider, provider_output_dir, export_name, listen)?; - wait_for_socket_listener(bridge_proxy_probe_addr(listen)).await?; - bridge_proxies.insert( - key, - BridgeProxyHandle { - child, - export_name: export_name.to_string(), - listen, - }, - ); - Ok(listen.port()) -} - -pub(super) fn spawn_bridge_proxy( - run_root: &Path, - provider: &LaunchedSite, - provider_output_dir: &Path, - export_name: &str, - listen: SocketAddr, -) -> Result { - let logs_root = run_root.join("bridge-proxies"); - fs::create_dir_all(&logs_root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", logs_root.display()))?; - let log_path = logs_root.join(format!("{export_name}.log")); - spawn_detached_child(run_root, &log_path, |cmd| { - cmd.arg("proxy") - .arg(provider_output_dir) - .arg("--export") - .arg(bridge_proxy_export_binding(export_name, listen)); - if provider.receipt.kind == SiteKind::Kubernetes { - let control = provider.router_control.to_string(); - cmd.arg("--router-addr") - .arg(provider.router_addr.to_string()) - .arg("--router-control-addr") - .arg(control); - } - }) -} - -pub(super) fn bridge_proxy_export_binding(export_name: &str, listen: SocketAddr) -> String { - format!("{export_name}={}:{}", listen.ip(), listen.port()) -} - -pub(super) fn bridge_proxy_bind_addr(consumer_kind: SiteKind, port: u16) -> SocketAddr { - host_service_bind_addr_for_consumer(consumer_kind, port) -} - -pub(super) fn bridge_proxy_probe_addr(listen: SocketAddr) -> SocketAddr { - listener_probe_addr(listen) -} - -pub(super) fn bridge_proxy_external_url( - port: u16, - protocol: NetworkProtocol, - consumer_kind: SiteKind, -) -> Result { - let host = host_service_host_for_consumer(consumer_kind); - Ok(match protocol { - NetworkProtocol::Http | NetworkProtocol::Https => format!("http://{host}:{port}"), - NetworkProtocol::Tcp => format!("tcp://{host}:{port}"), - _ => { - return Err(miette::miette!( - "mixed-site bridge proxy does not support protocol `{protocol}`" - )); - } - }) -} - -pub(crate) fn host_service_host_for_consumer(consumer_kind: SiteKind) -> String { - match consumer_kind { - SiteKind::Compose => CONTAINER_HOST_ALIAS.to_string(), - SiteKind::Direct | SiteKind::Vm | SiteKind::Kubernetes => { - container_host_for_consumer(SiteKind::Direct, consumer_kind) - } - } + external_slot_url(provider, provider_output_dir, link, consumer_kind) } pub(super) fn consumer_needs_host_wide_listener(consumer_kind: SiteKind) -> bool { - matches!(consumer_kind, SiteKind::Compose | SiteKind::Kubernetes) + matches!( + consumer_kind, + SiteKind::Compose | SiteKind::Kubernetes | SiteKind::Vm + ) || (cfg!(target_os = "linux") && matches!(consumer_kind, SiteKind::Direct)) } pub(crate) fn host_service_bind_addr_for_consumer( @@ -2056,9 +1982,23 @@ pub(super) fn external_slot_url( link: &RunLink, consumer_kind: SiteKind, ) -> Result { - let host = container_host_for_consumer(provider.receipt.kind, consumer_kind); + let router_mesh_addr = if let Some(router_mesh_addr) = + published_router_mesh_addr_for_consumer_kind(&provider.receipt, consumer_kind) + { + router_mesh_addr.to_string() + } else { + amber_site_controller::router_mesh_addr_for_consumer( + provider.receipt.kind, + consumer_kind, + provider + .receipt + .router_mesh_addr + .as_deref() + .ok_or_else(|| miette::miette!("provider site is missing router mesh addr"))?, + )? + }; let route_id = provider_export_route_id(provider_output_dir, link)?; - let mut mesh_url = Url::parse(&format!("mesh://{}:{}", host, provider.router_addr.port())) + let mut mesh_url = Url::parse(&format!("mesh://{router_mesh_addr}")) .into_diagnostic() .wrap_err("failed to build mesh link url")?; let peer_key = @@ -2095,21 +2035,13 @@ pub(super) fn container_host_for_consumer( } pub(super) fn container_host_from_resolved_ip( - provider_kind: SiteKind, + _provider_kind: SiteKind, consumer_kind: SiteKind, container_host_ip: Option<&str>, ) -> String { match consumer_kind { SiteKind::Direct | SiteKind::Vm => "127.0.0.1".to_string(), - SiteKind::Compose => { - if provider_kind == SiteKind::Kubernetes { - container_host_ip - .unwrap_or(CONTAINER_HOST_ALIAS) - .to_string() - } else { - CONTAINER_HOST_ALIAS.to_string() - } - } + SiteKind::Compose => CONTAINER_HOST_ALIAS.to_string(), SiteKind::Kubernetes => container_host_ip .unwrap_or(CONTAINER_HOST_ALIAS) .to_string(), @@ -2180,3 +2112,200 @@ pub(super) fn mesh_protocol(protocol: NetworkProtocol) -> Result { } }) } + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, fs, net::SocketAddr}; + + use amber_compiler::mesh::{PROXY_METADATA_FILENAME, PROXY_METADATA_VERSION}; + use amber_mesh::{MeshIdentity, MeshIdentityPublic, router_export_route_id}; + + use super::*; + + #[test] + fn launch_env_includes_process_dev_image_tag_overrides() { + unsafe { + std::env::set_var( + amber_images::DEV_IMAGE_TAGS_ENV, + "router=dev-tag,helper=dev-tag", + ); + } + let env = launch_env( + "run-test", + "scope", + SiteKind::Kubernetes, + &BTreeMap::new(), + &BTreeMap::new(), + None, + ) + .expect("launch env should build"); + assert_eq!( + env.get(amber_images::DEV_IMAGE_TAGS_ENV) + .map(String::as_str), + Some("router=dev-tag,helper=dev-tag"), + ); + unsafe { + std::env::remove_var(amber_images::DEV_IMAGE_TAGS_ENV); + } + } + + #[test] + fn cross_site_container_consumers_resolve_links_as_mesh_router_urls() { + let temp = tempfile::tempdir().expect("tempdir should create"); + fs::write( + temp.path().join(PROXY_METADATA_FILENAME), + serde_json::to_vec_pretty(&serde_json::json!({ + "version": PROXY_METADATA_VERSION, + "exports": { + "compose_http": { + "component": "/job-kind/compose_helper", + "provide": "http", + "capability_kind": "http", + "protocol": "http", + "router_mesh_port": 24000, + "route_id": router_export_route_id("compose_http", amber_mesh::MeshProtocol::Http), + } + } + })) + .expect("proxy metadata should serialize"), + ) + .expect("proxy metadata should write"); + + let router_identity = MeshIdentity::generate("/site/compose_local/router", None); + let provider = LaunchedSite { + receipt: SiteReceipt { + kind: SiteKind::Compose, + artifact_dir: temp.path().display().to_string(), + supervisor_pid: 0, + process_pid: None, + compose_project: Some("compose-test".to_string()), + kubernetes_namespace: None, + port_forward_pid: None, + context: None, + router_control: None, + router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: Some("host.docker.internal:24000".to_string()), + kubernetes_consumer_router_mesh_addr: Some("192.168.65.254:24000".to_string()), + router_identity_id: Some(router_identity.id.clone()), + router_public_key_b64: None, + site_controller_pid: None, + site_controller_url: None, + }, + router_control: ControlEndpoint::Tcp("127.0.0.1:24100".to_string()), + router_identity: MeshIdentityPublic::from_identity(&router_identity), + router_addr: SocketAddr::from(([127, 0, 0, 1], 24000)), + }; + let link = amber_compiler::run_plan::RunLink { + provider_site: "compose_local".to_string(), + consumer_site: "kind_local".to_string(), + provider_component: "/job-kind/compose_helper".to_string(), + provide: "http".to_string(), + consumer_component: "/job-kind/root".to_string(), + slot: "compose".to_string(), + weak: false, + protocol: NetworkProtocol::Http, + export_name: "compose_http".to_string(), + external_slot_name: "amber_link_compose_http".to_string(), + }; + + for consumer_kind in [SiteKind::Compose, SiteKind::Kubernetes] { + let resolved = tokio::runtime::Runtime::new() + .expect("tokio runtime should create") + .block_on(resolve_link_external_url_for_output( + &provider, + temp.path(), + &link, + consumer_kind, + temp.path(), + )) + .expect("container consumer link should resolve"); + + assert!( + resolved.starts_with("mesh://"), + "cross-site {consumer_kind:?} consumers should route through mesh, got {resolved}" + ); + assert!( + !resolved.starts_with("http://"), + "cross-site {consumer_kind:?} consumers should not bounce through an HTTP bridge \ + proxy: {resolved}" + ); + assert!(resolved.contains("peer_id=%2Fsite%2Fcompose_local%2Frouter")); + } + } + + #[test] + fn vm_site_consumers_keep_router_mesh_loopback_for_cross_site_links() { + let temp = tempfile::tempdir().expect("tempdir should create"); + fs::write( + temp.path().join(PROXY_METADATA_FILENAME), + serde_json::to_vec_pretty(&serde_json::json!({ + "version": PROXY_METADATA_VERSION, + "exports": { + "compose_http": { + "component": "/job-vm/compose_helper", + "provide": "http", + "capability_kind": "http", + "protocol": "http", + "router_mesh_port": 24000, + "route_id": router_export_route_id("compose_http", amber_mesh::MeshProtocol::Http), + } + } + })) + .expect("proxy metadata should serialize"), + ) + .expect("proxy metadata should write"); + + let router_identity = MeshIdentity::generate("/site/compose_local/router", None); + let provider = LaunchedSite { + receipt: SiteReceipt { + kind: SiteKind::Compose, + artifact_dir: temp.path().display().to_string(), + supervisor_pid: 0, + process_pid: None, + compose_project: Some("compose-test".to_string()), + kubernetes_namespace: None, + port_forward_pid: None, + context: None, + router_control: None, + router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: Some("host.docker.internal:24000".to_string()), + kubernetes_consumer_router_mesh_addr: Some("192.168.65.254:24000".to_string()), + router_identity_id: Some(router_identity.id.clone()), + router_public_key_b64: None, + site_controller_pid: None, + site_controller_url: None, + }, + router_control: ControlEndpoint::Tcp("127.0.0.1:24100".to_string()), + router_identity: MeshIdentityPublic::from_identity(&router_identity), + router_addr: SocketAddr::from(([127, 0, 0, 1], 24000)), + }; + let link = amber_compiler::run_plan::RunLink { + provider_site: "compose_local".to_string(), + consumer_site: "vm_local".to_string(), + provider_component: "/job-vm/compose_helper".to_string(), + provide: "http".to_string(), + consumer_component: "/job-vm/root".to_string(), + slot: "compose".to_string(), + weak: false, + protocol: NetworkProtocol::Http, + export_name: "compose_http".to_string(), + external_slot_name: "amber_link_compose_http".to_string(), + }; + + let resolved = tokio::runtime::Runtime::new() + .expect("tokio runtime should create") + .block_on(resolve_link_external_url_for_output( + &provider, + temp.path(), + &link, + SiteKind::Vm, + temp.path(), + )) + .expect("vm site consumer link should resolve"); + + assert!( + resolved.starts_with("mesh://127.0.0.1:24000"), + "vm site consumers should keep the live router mesh address, got {resolved}" + ); + } +} diff --git a/cli/src/mixed_run/tests.rs b/cli/src/mixed_run/tests.rs index fa45191e..a9b8d3de 100644 --- a/cli/src/mixed_run/tests.rs +++ b/cli/src/mixed_run/tests.rs @@ -52,6 +52,30 @@ fn site_controller_image_override_uses_dev_tag() { ); } +#[test] +fn recorded_process_roots_include_site_supervisor() { + let site = SiteReceipt { + kind: SiteKind::Direct, + artifact_dir: "/tmp/artifact".to_string(), + supervisor_pid: 7, + process_pid: Some(11), + compose_project: None, + kubernetes_namespace: None, + port_forward_pid: Some(13), + context: None, + router_control: None, + router_mesh_addr: None, + compose_consumer_router_mesh_addr: None, + kubernetes_consumer_router_mesh_addr: None, + router_identity_id: None, + router_public_key_b64: None, + site_controller_pid: None, + site_controller_url: None, + }; + + assert_eq!(recorded_process_roots(&site), vec![7, 11, 13]); +} + #[test] fn site_controller_local_router_control_uses_backend_local_control_targets() { let artifact_dir = Path::new("/tmp/site-artifact"); @@ -96,6 +120,27 @@ fn site_controller_peer_router_urls_are_local_to_the_controller_site() { ); } +#[test] +fn compose_consumers_use_host_alias_for_kubernetes_router_mesh() { + assert_eq!( + supervisor::container_host_from_resolved_ip( + SiteKind::Kubernetes, + SiteKind::Compose, + Some("192.168.65.254"), + ), + "host.docker.internal" + ); + assert_eq!( + amber_site_controller::router_mesh_addr_for_consumer( + SiteKind::Kubernetes, + SiteKind::Compose, + "127.0.0.1:24077", + ) + .expect("compose consumers should be able to route to kubernetes peers"), + "host.docker.internal:24077" + ); +} + #[test] fn site_controller_image_includes_the_amber_cli_binary() { let dockerfile = fs::read_to_string( @@ -190,6 +235,29 @@ fn local_site_controller_addr_requires_loopback_http() { ); } +#[test] +fn host_service_bind_addr_matches_component_reachability() { + assert_eq!( + supervisor::host_service_bind_addr_for_consumer(SiteKind::Compose, 24200), + SocketAddr::from(([0, 0, 0, 0], 24200)) + ); + assert_eq!( + supervisor::host_service_bind_addr_for_consumer(SiteKind::Kubernetes, 24201), + SocketAddr::from(([0, 0, 0, 0], 24201)) + ); + assert_eq!( + supervisor::host_service_bind_addr_for_consumer(SiteKind::Vm, 24202), + SocketAddr::from(([0, 0, 0, 0], 24202)) + ); + + let direct = supervisor::host_service_bind_addr_for_consumer(SiteKind::Direct, 24203); + if cfg!(target_os = "linux") { + assert_eq!(direct, SocketAddr::from(([0, 0, 0, 0], 24203))); + } else { + assert_eq!(direct, SocketAddr::from(([127, 0, 0, 1], 24203))); + } +} + #[test] fn local_site_controller_ready_waits_for_http_listener() { let listener = TcpListener::bind(("127.0.0.1", 0)).expect("listener should bind"); diff --git a/cli/src/tests.rs b/cli/src/tests.rs index 0a6bf2be..771c1849 100644 --- a/cli/src/tests.rs +++ b/cli/src/tests.rs @@ -888,6 +888,77 @@ async fn cleanup_direct_runtime_removes_partial_startup_artifacts() { ); } +#[cfg(unix)] +#[tokio::test] +async fn cleanup_direct_runtime_terminates_wrapper_descendants() { + let plan_root = tempfile::tempdir().expect("temp dir should be created"); + let runtime_dir = tempfile::Builder::new() + .prefix("amber-direct-test-") + .tempdir() + .expect("runtime dir should be created"); + let runtime_root = runtime_dir.path().to_path_buf(); + let runtime_state_path = direct_runtime_state_path(plan_root.path()); + fs::create_dir_all(runtime_state_path.parent().expect("state parent")) + .expect("state parent should be created"); + fs::write(&runtime_state_path, "{}").expect("state file should be written"); + + let child_pid_path = runtime_root.join("child.pid"); + let child = TokioCommand::new("sh") + .arg("-c") + .arg(format!( + "setsid sleep 30 >/dev/null 2>&1 & echo $! > {} ; wait", + child_pid_path.display() + )) + .spawn() + .expect("wrapper should spawn"); + let mut children = vec![ManagedChild { + name: "wrapper-with-descendant".to_string(), + wrapper: Some(child), + #[cfg(target_os = "linux")] + wrapper_pid: 0, + #[cfg(target_os = "linux")] + managed_pid: 0, + }]; + #[cfg(target_os = "linux")] + { + let pid = children[0] + .wrapper + .as_ref() + .and_then(tokio::process::Child::id) + .expect("child pid should be available"); + children[0].wrapper_pid = pid; + children[0].managed_pid = pid; + } + + let deadline = Instant::now() + Duration::from_secs(5); + while !child_pid_path.is_file() && Instant::now() < deadline { + sleep(Duration::from_millis(25)).await; + } + let child_pid = fs::read_to_string(&child_pid_path) + .expect("descendant pid should be recorded") + .trim() + .parse::() + .expect("descendant pid should parse"); + + cleanup_direct_runtime( + &mut children, + Vec::new(), + &runtime_state_path, + None, + Some(runtime_dir), + ) + .await; + + let stopped = !crate::unix_process::pid_is_alive(child_pid); + if !stopped { + let _ = unsafe { libc::kill(child_pid as i32, libc::SIGKILL) }; + } + assert!( + stopped, + "cleanup should terminate descendants that outlive their wrapper" + ); +} + #[test] fn write_direct_runtime_state_preserves_projected_router_mesh_port() { let plan_root = tempfile::tempdir().expect("temp dir should be created"); diff --git a/cli/src/unix_process.rs b/cli/src/unix_process.rs new file mode 100644 index 00000000..867f2b74 --- /dev/null +++ b/cli/src/unix_process.rs @@ -0,0 +1,185 @@ +#[cfg(unix)] +use std::{ + collections::{BTreeSet, HashMap}, + process::Command, +}; + +#[cfg(unix)] +use miette::{Context as _, IntoDiagnostic as _, Result}; +#[cfg(unix)] +use tokio::time::{Duration, Instant, sleep}; + +#[cfg(unix)] +pub(crate) fn pid_is_alive(pid: u32) -> bool { + let alive = unsafe { + libc::kill(pid as i32, 0) == 0 + || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) + }; + alive && process_status_code(pid) != Some('Z') +} + +#[cfg(unix)] +pub(crate) async fn terminate_process_roots(root_pids: &[u32], timeout: Duration) -> Result<()> { + let mut seen = BTreeSet::new(); + let mut ordered = Vec::new(); + for root_pid in root_pids { + for pid in process_tree_postorder(*root_pid)? { + if seen.insert(pid) { + ordered.push(pid); + } + } + } + if ordered.is_empty() { + return Ok(()); + } + + send_signal_to_pids(&ordered, libc::SIGTERM); + wait_for_pids_exit(&ordered, timeout).await; + + let survivors = ordered + .iter() + .copied() + .filter(|pid| pid_is_alive(*pid)) + .collect::>(); + if survivors.is_empty() { + return Ok(()); + } + + send_signal_to_pids(&survivors, libc::SIGKILL); + wait_for_pids_exit(&survivors, Duration::from_secs(2)).await; + Ok(()) +} + +#[cfg(unix)] +fn process_status_code(pid: u32) -> Option { + let output = Command::new("ps") + .arg("-o") + .arg("stat=") + .arg("-p") + .arg(pid.to_string()) + .output() + .ok()?; + if !output.status.success() { + return None; + } + parse_process_status_code(&String::from_utf8_lossy(&output.stdout)) +} + +#[cfg(unix)] +fn parse_process_status_code(raw: &str) -> Option { + raw.split_whitespace() + .next()? + .chars() + .next() + .map(|state| state.to_ascii_uppercase()) +} + +#[cfg(unix)] +fn send_signal_to_pids(pids: &[u32], signal: i32) { + for pid in pids { + unsafe { + libc::kill(*pid as i32, signal); + } + } +} + +#[cfg(unix)] +async fn wait_for_pids_exit(pids: &[u32], timeout: Duration) { + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if pids.iter().all(|pid| !pid_is_alive(*pid)) { + return; + } + sleep(Duration::from_millis(100)).await; + } +} + +#[cfg(unix)] +fn process_tree_postorder(root_pid: u32) -> Result> { + let output = Command::new("ps") + .arg("-axo") + .arg("pid=,ppid=") + .output() + .into_diagnostic() + .wrap_err("failed to enumerate process tree")?; + if !output.status.success() { + return Err(miette::miette!( + "failed to enumerate process tree: status {}", + output.status + )); + } + + let parent_by_pid = parse_process_table(&String::from_utf8_lossy(&output.stdout))?; + if !parent_by_pid.contains_key(&root_pid) && !pid_is_alive(root_pid) { + return Ok(Vec::new()); + } + + let mut children_by_parent = HashMap::>::new(); + for (pid, ppid) in parent_by_pid { + children_by_parent.entry(ppid).or_default().push(pid); + } + + let mut ordered = Vec::new(); + collect_process_tree_postorder(root_pid, &children_by_parent, &mut ordered); + Ok(ordered) +} + +#[cfg(unix)] +fn collect_process_tree_postorder( + pid: u32, + children_by_parent: &HashMap>, + ordered: &mut Vec, +) { + if let Some(children) = children_by_parent.get(&pid) { + for child in children { + collect_process_tree_postorder(*child, children_by_parent, ordered); + } + } + ordered.push(pid); +} + +#[cfg(unix)] +fn parse_process_table(raw: &str) -> Result> { + let mut parent_by_pid = HashMap::new(); + for line in raw.lines() { + let mut fields = line.split_whitespace(); + let Some(pid) = fields.next() else { + continue; + }; + let Some(ppid) = fields.next() else { + continue; + }; + let pid = pid + .parse::() + .into_diagnostic() + .wrap_err_with(|| format!("invalid process table pid `{pid}`"))?; + let ppid = ppid + .parse::() + .into_diagnostic() + .wrap_err_with(|| format!("invalid process table parent pid `{ppid}`"))?; + parent_by_pid.insert(pid, ppid); + } + Ok(parent_by_pid) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + #[test] + fn parse_process_table_reads_pid_parent_pairs() { + let table = parse_process_table("10 1\n11 10\n").expect("process table should parse"); + assert_eq!(table.get(&10), Some(&1)); + assert_eq!(table.get(&11), Some(&10)); + } + + #[test] + fn collect_process_tree_postorder_lists_descendants_before_root() { + let tree = HashMap::from([(1, vec![2, 3]), (2, vec![4])]); + let mut ordered = Vec::new(); + collect_process_tree_postorder(1, &tree, &mut ordered); + assert_eq!(ordered, vec![4, 2, 3, 1]); + } +} diff --git a/cli/tests/direct_smoke.rs b/cli/tests/direct_smoke.rs index ad3a9c1b..a9f762bf 100644 --- a/cli/tests/direct_smoke.rs +++ b/cli/tests/direct_smoke.rs @@ -1,6 +1,9 @@ +#[path = "test_support/port_allocator.rs"] +mod port_allocator_support; + use std::{ env, fs, - net::{SocketAddr, TcpListener}, + net::SocketAddr, path::Path, process::{Command, Stdio}, thread, @@ -8,11 +11,11 @@ use std::{ }; use amber_mesh::{MeshConfig, MeshIdentity, TransportConfig, encode_config_b64}; +use port_allocator_support::reserve_test_loopback_port; use serde_json::Value; fn pick_free_port() -> u16 { - let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).unwrap(); - listener.local_addr().unwrap().port() + reserve_test_loopback_port() } fn workspace_root() -> std::path::PathBuf { diff --git a/cli/tests/linux_vm.rs b/cli/tests/linux_vm.rs index d30df662..b73f5f53 100644 --- a/cli/tests/linux_vm.rs +++ b/cli/tests/linux_vm.rs @@ -6,6 +6,8 @@ mod cloud_image_support; mod macos_vm_support; #[path = "test_support/outputs_root.rs"] mod outputs_root_support; +#[path = "test_support/port_allocator.rs"] +mod port_allocator_support; #[path = "test_support/workspace_root.rs"] mod workspace_root_support; @@ -13,7 +15,6 @@ use std::{ collections::BTreeMap, env, fs, hash::{Hash as _, Hasher as _}, - net::{SocketAddr, TcpListener}, path::{Path, PathBuf}, process::{Child, Command, Output, Stdio}, thread, @@ -23,6 +24,7 @@ use std::{ use cloud_image_support::default_host_arch_cloud_image_filename; use macos_vm_support::resolve_aarch64_firmware; use outputs_root_support::cli_test_outputs_root; +use port_allocator_support::reserve_test_loopback_port; use tempfile::TempDir; use workspace_root_support::workspace_root; @@ -35,6 +37,10 @@ const LINUX_VM_KIND_VERSION: &str = "v0.26.0"; const LINUX_VM_KIND_NODE_IMAGE: &str = "kindest/node:v1.32.0@sha256:c48c62eac5da28cdadcf560d1d8616cfa6783b58f0d94cf63ad1bf49600cb027"; const LINUX_VM_KIND_PULL_TIMEOUT: &str = "180s"; +const LINUX_VM_DIRECT_SMOKE_FILTER_ENV: &str = "AMBER_LINUX_VM_DIRECT_SMOKE_FILTER"; +const LINUX_VM_DYNAMIC_CAPABILITIES_FILTER_ENV: &str = "AMBER_LINUX_VM_DYNAMIC_CAPABILITIES_FILTER"; +const LINUX_VM_FRAMEWORK_COMPONENT_FILTER_ENV: &str = "AMBER_LINUX_VM_FRAMEWORK_COMPONENT_FILTER"; +const LINUX_VM_MIXED_RUN_FILTER_ENV: &str = "AMBER_LINUX_VM_MIXED_RUN_FILTER"; struct GuestArch { cloud_image_filename: &'static str, @@ -257,8 +263,7 @@ fn qemu_system_binary(arch: &GuestArch) -> PathBuf { } fn pick_free_port() -> u16 { - let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).unwrap(); - listener.local_addr().unwrap().port() + reserve_test_loopback_port() } fn shell_escape(text: &str) -> String { @@ -958,10 +963,6 @@ fn ensure_provisioned_image(profile: ProvisionProfile) -> Result Result<(), String> { - run_linux_guest_mixed_run_filter(test_name) -} - fn run_linux_guest_mixed_run_filter(test_filter: &str) -> Result<(), String> { run_linux_guest_test( format!("linux-vm-{test_filter}-"), @@ -973,6 +974,15 @@ fn run_linux_guest_mixed_run_filter(test_filter: &str) -> Result<(), String> { ) } +// Keep the Linux VM live surface to one guest boot per test family. Narrow reruns can still use +// the same aggregate test by overriding its guest-side filter env var. +fn linux_vm_guest_filter(env_var: &str, default: &str) -> String { + env::var(env_var) + .ok() + .filter(|filter| !filter.trim().is_empty()) + .unwrap_or_else(|| default.to_owned()) +} + fn run_linux_guest_direct_smoke_test(test_name: &str) -> Result<(), String> { run_linux_guest_test( format!("linux-vm-{test_name}-"), @@ -1000,12 +1010,10 @@ fn linux_vm_runs_vm_smoke_test() { #[ignore = "requires qemu on macOS; boots Ubuntu and runs the real Linux mixed_run tests inside \ the guest"] fn linux_vm_runs_mixed_run_tests() { - run_linux_guest_test( - "linux-vm-mixed-run-", - ProvisionProfile::MixedRun, - "cargo test -p amber-cli --test mixed_run mixed_run_ -- --ignored --nocapture \ - --test-threads=1", - ) + run_linux_guest_mixed_run_filter(&linux_vm_guest_filter( + LINUX_VM_MIXED_RUN_FILTER_ENV, + "mixed_run_", + )) .unwrap_or_else(|err| panic!("{err}")); } @@ -1013,227 +1021,35 @@ fn linux_vm_runs_mixed_run_tests() { #[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component live tests \ inside the guest"] fn linux_vm_runs_framework_component_live_tests() { - run_linux_guest_mixed_run_filter("framework_component_").unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component direct \ - create/destroy live test inside the guest"] -fn linux_vm_runs_framework_component_direct_create_destroy_live() { - run_linux_guest_mixed_run_test("framework_component_direct_create_destroy_live") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component bounded \ - template replay live test inside the guest"] -fn linux_vm_runs_framework_component_bounded_template_frozen_source_replay_live() { - run_linux_guest_mixed_run_test( - "framework_component_bounded_template_frozen_source_replay_live", - ) + run_linux_guest_mixed_run_filter(&linux_vm_guest_filter( + LINUX_VM_FRAMEWORK_COMPONENT_FILTER_ENV, + "framework_component_", + )) .unwrap_or_else(|err| panic!("{err}")); } -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component \ - cross-backend matrix live test inside the guest"] -fn linux_vm_runs_framework_component_cross_backend_matrix_live() { - run_linux_guest_mixed_run_test("framework_component_cross_backend_matrix_live") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component \ - delegated-realm cross-site live test inside the guest"] -fn linux_vm_runs_framework_component_delegated_realm_cross_site_live() { - run_linux_guest_mixed_run_test("framework_component_delegated_realm_cross_site_live") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component \ - direct-parent compose-child live test inside the guest"] -fn linux_vm_runs_framework_component_direct_parent_compose_child_live() { - run_linux_guest_mixed_run_test("framework_component_direct_parent_compose_child_live") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component \ - dynamic-child teardown live test inside the guest"] -fn linux_vm_runs_framework_component_dynamic_children_teardown_with_run_live() { - run_linux_guest_mixed_run_test("framework_component_dynamic_children_teardown_with_run_live") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component kind \ - creator after compose churn live test inside the guest"] -fn linux_vm_runs_framework_component_kind_creator_after_compose_churn_live() { - run_linux_guest_mixed_run_test("framework_component_kind_creator_after_compose_churn_live") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component kind \ - creator compose-child live test inside the guest"] -fn linux_vm_runs_framework_component_kind_creator_compose_child_live() { - run_linux_guest_mixed_run_test("framework_component_kind_creator_compose_child_live") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux framework_component kind \ - root-export live test inside the guest"] -fn linux_vm_runs_framework_component_kind_root_export_live() { - run_linux_guest_mixed_run_test("framework_component_kind_root_export_live") - .unwrap_or_else(|err| panic!("{err}")); -} - #[test] #[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux dynamic-capability live tests \ inside the guest"] fn linux_vm_runs_dynamic_capability_live_tests() { - run_linux_guest_mixed_run_filter("dynamic_capabilities_").unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux dynamic-capability dynamic \ - child post-create share live test inside the guest"] -fn linux_vm_runs_dynamic_capabilities_dynamic_child_post_create_share_live() { - run_linux_guest_mixed_run_test("dynamic_capabilities_dynamic_child_post_create_share_live") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux dynamic-capability manual \ - materialization live test inside the guest"] -fn linux_vm_runs_dynamic_capabilities_manual_materialization_live() { - run_linux_guest_mixed_run_test("dynamic_capabilities_manual_materialization_live") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux dynamic-capability external \ - slot root share live test inside the guest"] -fn linux_vm_runs_dynamic_capabilities_external_slot_root_share_live() { - run_linux_guest_mixed_run_test("dynamic_capabilities_external_slot_root_share_live") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux dynamic-capability snapshot \ - replay dynamic child live test inside the guest"] -fn linux_vm_runs_dynamic_capabilities_snapshot_replay_dynamic_child_live() { - run_linux_guest_mixed_run_test("dynamic_capabilities_snapshot_replay_dynamic_child_live") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the real Linux mixed_run cleanup test \ - inside the guest"] -fn linux_vm_runs_mixed_run_cleanup_after_coordinator_dies_during_setup() { - run_linux_guest_mixed_run_test("mixed_run_cleanup_after_coordinator_dies_during_setup") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the real Linux mixed_run direct/compose \ - proxy smoke test inside the guest"] -fn linux_vm_runs_mixed_run_direct_compose_proxy_smoke() { - run_linux_guest_mixed_run_test("mixed_run_direct_compose_proxy_smoke") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the real Linux mixed_run detached stop \ - smoke test inside the guest"] -fn linux_vm_runs_mixed_run_detached_stop_smoke() { - run_linux_guest_mixed_run_test("mixed_run_detached_stop_smoke") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the real Linux mixed_run five-site \ - startup test inside the guest"] -fn linux_vm_runs_mixed_run_five_site_startup_state_and_teardown() { - run_linux_guest_mixed_run_test("mixed_run_five_site_startup_state_and_teardown") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the real Linux mixed_run local \ - observability smoke test inside the guest"] -fn linux_vm_runs_mixed_run_local_observability_scenario_smoke() { - run_linux_guest_mixed_run_test("mixed_run_local_observability_scenario_smoke") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the real Linux mixed_run recovery test \ - inside the guest"] -fn linux_vm_runs_mixed_run_recovers_direct_component_failure_after_setup() { - run_linux_guest_mixed_run_test("mixed_run_recovers_direct_component_failure_after_setup") - .unwrap_or_else(|err| panic!("{err}")); + run_linux_guest_mixed_run_filter(&linux_vm_guest_filter( + LINUX_VM_DYNAMIC_CAPABILITIES_FILTER_ENV, + "dynamic_capabilities_", + )) + .unwrap_or_else(|err| panic!("{err}")); } #[test] #[ignore = "requires qemu on macOS; boots Ubuntu and runs the real Linux direct smoke tests inside \ the guest"] fn linux_vm_runs_direct_smoke_tests() { - run_linux_guest_direct_smoke_test("direct_smoke_").unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux direct smoke http startup test \ - inside the guest"] -fn linux_vm_runs_direct_smoke_python_http_server_starts_and_stops() { - run_linux_guest_direct_smoke_test("direct_smoke_python_http_server_starts_and_stops") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux direct smoke relative-path \ - test inside the guest"] -fn linux_vm_runs_direct_smoke_resolves_relative_program_path_from_manifest_dir() { - run_linux_guest_direct_smoke_test( - "direct_smoke_resolves_relative_program_path_from_manifest_dir", - ) + run_linux_guest_direct_smoke_test(&linux_vm_guest_filter( + LINUX_VM_DIRECT_SMOKE_FILTER_ENV, + "direct_smoke_", + )) .unwrap_or_else(|err| panic!("{err}")); } -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux direct smoke helper-mount test \ - inside the guest"] -fn linux_vm_runs_direct_smoke_mount_under_run_with_helper() { - run_linux_guest_direct_smoke_test("direct_smoke_mount_under_run_with_helper") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux direct smoke storage upgrade \ - test inside the guest"] -fn linux_vm_runs_direct_smoke_storage_persists_across_upgrade() { - run_linux_guest_direct_smoke_test("direct_smoke_storage_persists_across_upgrade") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux direct smoke env scrub test \ - inside the guest"] -fn linux_vm_runs_direct_smoke_does_not_leak_host_env_into_component() { - run_linux_guest_direct_smoke_test("direct_smoke_does_not_leak_host_env_into_component") - .unwrap_or_else(|err| panic!("{err}")); -} - -#[test] -#[ignore = "requires qemu on macOS; boots Ubuntu and runs the Linux direct smoke host-fs isolation \ - test inside the guest"] -fn linux_vm_runs_direct_smoke_blocks_host_file_reads_outside_allowed_mounts() { - run_linux_guest_direct_smoke_test("direct_smoke_blocks_host_file_reads_outside_allowed_mounts") - .unwrap_or_else(|err| panic!("{err}")); -} - #[test] fn provisioned_cache_keys_track_guest_setup() { let vm_smoke_cache_key = ProvisionProfile::VmSmoke.cache_key(); diff --git a/cli/tests/mixed_run/smoke.rs b/cli/tests/mixed_run/smoke.rs index 9f5d6338..02e82260 100644 --- a/cli/tests/mixed_run/smoke.rs +++ b/cli/tests/mixed_run/smoke.rs @@ -743,26 +743,7 @@ fn framework_control_state_path(run: &RunHandle) -> PathBuf { } fn read_framework_control_state(control_state_root: &Path) -> Value { - let mut live_children = Vec::new(); - let entries = fs::read_dir(control_state_root) - .unwrap_or_else(|err| panic!("failed to read {}: {err}", control_state_root.display())); - for entry in entries.filter_map(Result::ok) { - let Ok(file_type) = entry.file_type() else { - continue; - }; - if !file_type.is_dir() { - continue; - } - let state_path = entry.path().join("site-controller-state.json"); - if !state_path.is_file() { - continue; - } - let state = read_json(&state_path); - if let Some(children) = state["live_children"].as_array() { - live_children.extend(children.iter().cloned()); - } - } - json!({ "live_children": live_children }) + framework_control_state_snapshot(control_state_root) } fn framework_site_controller_plan_path(run: &RunHandle) -> PathBuf { @@ -816,25 +797,26 @@ fn framework_controller_post(run: &RunHandle, path: &str, payload: &Value) -> (u .as_str() .expect("site controller plan should publish auth token"); let body = serde_json::to_string(payload).expect("request body should serialize"); - let output = std::process::Command::new("curl") - .arg("-sS") - .arg("--max-time") - .arg("30.000") - .arg("-X") - .arg("POST") - .arg("-H") - .arg("content-type: application/json") - .arg("-H") - .arg(format!("x-amber-framework-auth: {auth_token}")) - .arg("--data") - .arg(body) - .arg("-o") - .arg("-") - .arg("-w") - .arg("\n%{http_code}") - .arg(format!("{}{path}", authority_url.trim_end_matches('/'))) - .output() - .expect("site controller request should complete"); + let output = command_output_via_tempfiles( + std::process::Command::new("curl") + .arg("-sS") + .arg("--max-time") + .arg("30.000") + .arg("-X") + .arg("POST") + .arg("-H") + .arg("content-type: application/json") + .arg("-H") + .arg(format!("x-amber-framework-auth: {auth_token}")) + .arg("--data") + .arg(body) + .arg("-o") + .arg("-") + .arg("-w") + .arg("\n%{http_code}") + .arg(format!("{}{path}", authority_url.trim_end_matches('/'))), + "framework controller request", + ); let stdout = String::from_utf8_lossy(&output.stdout); let (body, status) = stdout .rsplit_once('\n') @@ -873,13 +855,7 @@ fn wait_for_live_child(control_state_path: &Path, name: &str) -> u64 { } fn framework_child_artifact(run: &RunHandle, site_id: &str, child_id: u64) -> PathBuf { - run.run_root - .join("state") - .join(site_id) - .join("framework-component") - .join("children") - .join(child_id.to_string()) - .join("artifact") + framework_child_artifact_dir(&run.run_root, site_id, child_id) } fn framework_manifest_url(path: &Path) -> String { @@ -923,9 +899,11 @@ fn spawn_framework_proxy_for_site( } fn wait_for_framework_child_absent( + run: &RunHandle, control_state_path: &Path, child_name: &str, - child_roots: &[PathBuf], + child_sites: &[&str], + child_id: u64, timeout: Duration, ) { wait_for_condition( @@ -934,7 +912,10 @@ fn wait_for_framework_child_absent( let no_live_child = read_framework_control_state(control_state_path)["live_children"] .as_array() .is_some_and(|children| children.iter().all(|child| child["name"] != child_name)); - no_live_child && child_roots.iter().all(|root| !root.exists()) + no_live_child + && child_sites + .iter() + .all(|site_id| framework_child_is_absent(&run.run_root, site_id, child_id)) }, &format!("dynamic child `{child_name}` removed from control state and site artifacts"), ); @@ -2277,15 +2258,16 @@ fn framework_component_create_to_unoffered_site_fails_deterministically_live() { ); let storage_root = temp.path().join("state"); - let output = amber_command() - .arg("run") - .arg(&manifest) - .arg("--placement") - .arg(&placement) - .arg("--storage-root") - .arg(&storage_root) - .output() - .expect("failed to run amber"); + let output = command_output_via_tempfiles( + amber_command() + .arg("run") + .arg(&manifest) + .arg("--placement") + .arg(&placement) + .arg("--storage-root") + .arg(&storage_root), + "amber run", + ); assert!( !output.status.success(), "run should fail deterministically when a bounded template allows a manifest that \ @@ -3003,12 +2985,11 @@ fn framework_component_delegated_realm_cross_site_live() { "delegated destroy should succeed; response: {destroy_response}" ); wait_for_framework_child_absent( + &run, &control_state_path, "sibling", - &[sibling_artifact - .parent() - .expect("sibling artifact should have a parent") - .to_path_buf()], + &["compose_local"], + sibling_id, Duration::from_secs(60), ); @@ -4015,15 +3996,10 @@ fn framework_component_kind_creator_after_compose_churn_live() { {create_response}" ); let child_id = wait_for_live_child(&control_state_path, template_case.child_name); - let child_roots = template_case + let child_sites = template_case .exports .iter() - .map(|(site_id, _, _)| { - framework_child_artifact(&run, site_id, child_id) - .parent() - .expect("dynamic child artifact should have a parent") - .to_path_buf() - }) + .map(|(site_id, _, _)| *site_id) .collect::>(); let (destroy_status, destroy_response) = framework_destroy_child_via_admin(compose_creator_port, template_case.child_name); @@ -4033,9 +4009,11 @@ fn framework_component_kind_creator_after_compose_churn_live() { template_case.child_name, ); wait_for_framework_child_absent( + &run, &control_state_path, template_case.child_name, - &child_roots, + &child_sites, + child_id, Duration::from_secs(300), ); } @@ -4088,6 +4066,23 @@ fn framework_component_kind_creator_after_compose_churn_live() { create_status, 200, "kind creator create after compose churn should succeed; response: {create_response}" ); + let child_id = wait_for_live_child(&control_state_path, "job-compose"); + + let root_artifact = framework_child_artifact(&run, "compose_local", child_id); + let root_port = pick_free_port(); + let mut root_proxy = spawn_framework_proxy_for_site( + &root_artifact, + "http", + root_port, + site_state("compose_local"), + ); + wait_for_path(&mut root_proxy, root_port, "/id", Duration::from_secs(300)); + assert_eq!( + wait_for_body(&mut root_proxy, root_port, "/id", Duration::from_secs(30)), + "child-compose-root", + "kind creator should expose the recreated compose child after compose churn" + ); + stop_proxy(&mut root_proxy); stop_proxy(&mut kind_creator_proxy); run.stop(); @@ -4193,15 +4188,10 @@ fn framework_component_cross_backend_matrix_live() { assert_eq!(create_json["child"]["name"], template_case.child_name); let child_id = wait_for_live_child(&control_state_path, template_case.child_name); - let child_roots = template_case + let child_sites = template_case .exports .iter() - .map(|(site_id, _, _)| { - framework_child_artifact(&run, site_id, child_id) - .parent() - .expect("dynamic child artifact should have a parent") - .to_path_buf() - }) + .map(|(site_id, _, _)| *site_id) .collect::>(); let root_site_id = template_case.exports[0].0; @@ -4256,9 +4246,11 @@ fn framework_component_cross_backend_matrix_live() { template_case.child_name, ); wait_for_framework_child_absent( + &run, &control_state_path, template_case.child_name, - &child_roots, + &child_sites, + child_id, Duration::from_secs(300), ); } diff --git a/cli/tests/mixed_run/support.rs b/cli/tests/mixed_run/support.rs index 40973eab..3697b5a2 100644 --- a/cli/tests/mixed_run/support.rs +++ b/cli/tests/mixed_run/support.rs @@ -4,6 +4,8 @@ mod cloud_image_support; #[path = "../test_support/outputs_root.rs"] mod outputs_root_support; +#[path = "../test_support/port_allocator.rs"] +mod port_allocator_support; #[path = "../test_support/target_dir.rs"] mod target_dir_support; #[path = "../test_support/workspace_root.rs"] @@ -11,7 +13,10 @@ mod workspace_root_support; use std::{ collections::BTreeSet, - env, fs, + env, + ffi::OsString, + fs, + hash::{DefaultHasher, Hash, Hasher}, io::{Read, Write}, net::{SocketAddr, TcpListener, TcpStream}, path::{Path, PathBuf}, @@ -24,9 +29,14 @@ use std::{ time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; -use amber_images::{AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER, AMBER_SITE_CONTROLLER}; +use amber_images::{ + AMBER_DOCKER_GATEWAY, AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER, AMBER_SITE_CONTROLLER, + DEV_IMAGE_TAGS_ENV, INTERNAL_IMAGE_OVERRIDE_KEYS, ImageRef, override_reference, + parse_dev_image_tag_overrides, +}; use cloud_image_support::default_host_arch_cloud_image_filename; use outputs_root_support::cli_test_outputs_root; +use port_allocator_support::reserve_test_loopback_port; use serde_json::{Value, json}; use target_dir_support::cargo_target_dir; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; @@ -145,6 +155,38 @@ pub(crate) struct SpawnedProxy { output_dir: PathBuf, } +#[derive(serde::Deserialize)] +struct StaleRunReceipt { + run_id: String, + #[serde(default)] + observability: Option, + #[serde(default)] + bridge_proxies: Vec, + sites: std::collections::BTreeMap, +} + +#[derive(serde::Deserialize)] +struct StaleObservabilityReceipt { + #[serde(default)] + sink_pid: Option, +} + +#[derive(serde::Deserialize)] +struct StaleBridgeProxyReceipt { + pid: u32, +} + +#[derive(serde::Deserialize)] +struct StaleSiteReceipt { + supervisor_pid: u32, + #[serde(default)] + process_pid: Option, + #[serde(default)] + port_forward_pid: Option, + #[serde(default)] + site_controller_pid: Option, +} + impl TestTempDir { pub(crate) fn path(&self) -> &Path { &self.path @@ -184,6 +226,7 @@ pub(crate) fn mixed_run_base_image() -> PathBuf { pub(crate) fn temp_output_dir(prefix: &str) -> TestTempDir { fs::create_dir_all(outputs_root()).expect("failed to create cli test outputs root"); + cleanup_abandoned_test_runs_once(); let temp = tempfile::Builder::new() .prefix(prefix) .tempdir_in(outputs_root()) @@ -200,41 +243,83 @@ pub(crate) fn temp_output_dir(prefix: &str) -> TestTempDir { } } -pub(crate) fn pick_free_port() -> u16 { - const TEST_PORT_RANGE_START: u16 = 20000; - const TEST_PORT_RANGE_END: u16 = 30000; - static RESERVED_TEST_PORTS: OnceLock>> = OnceLock::new(); - - let reserved = RESERVED_TEST_PORTS.get_or_init(|| Mutex::new(BTreeSet::new())); - let mut reserved = reserved - .lock() - .expect("test port allocator should not be poisoned"); - let span = u32::from(TEST_PORT_RANGE_END - TEST_PORT_RANGE_START); - let mut next = - TEST_PORT_RANGE_START + (std::process::id() % span) as u16 + reserved.len() as u16; - for _ in 0..usize::from(TEST_PORT_RANGE_END - TEST_PORT_RANGE_START) { - if next >= TEST_PORT_RANGE_END { - next = TEST_PORT_RANGE_START; +fn cleanup_abandoned_test_runs_once() { + static CLEANUP_ONCE: OnceLock<()> = OnceLock::new(); + CLEANUP_ONCE.get_or_init(|| { + for entry in fs::read_dir(outputs_root()) + .unwrap_or_else(|err| panic!("failed to read {}: {err}", outputs_root().display())) + { + let Ok(entry) = entry else { + continue; + }; + let storage_root = entry.path(); + if storage_root.is_dir() { + cleanup_abandoned_test_runs_in_storage_root(&storage_root); + } } - let port = next; - next += 1; - if reserved.contains(&port) { + }); +} + +fn cleanup_abandoned_test_runs_in_storage_root(storage_root: &Path) { + let runs_dir = storage_root.join("runs"); + let Ok(entries) = fs::read_dir(&runs_dir) else { + return; + }; + for entry in entries { + let Ok(entry) = entry else { + continue; + }; + let run_root = entry.path(); + let receipt_path = run_root.join("receipt.json"); + if !receipt_path.is_file() { continue; } - match TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], port))) { - Ok(listener) => { - drop(listener); - reserved.insert(port); - return port; - } - Err(_) => continue, + let Ok(receipt_bytes) = fs::read(&receipt_path) else { + continue; + }; + let Ok(receipt) = serde_json::from_slice::(&receipt_bytes) else { + continue; + }; + if stale_run_has_live_processes(&receipt) { + continue; } + let _ = command_output_via_tempfiles( + amber_command() + .arg("stop") + .arg(&receipt.run_id) + .arg("--storage-root") + .arg(storage_root), + "amber stop stale test run", + ); } - panic!( - "failed to allocate a unique mixed-run test port in {}-{}", - TEST_PORT_RANGE_START, - TEST_PORT_RANGE_END - 1 - ); +} + +fn stale_run_has_live_processes(receipt: &StaleRunReceipt) -> bool { + receipt + .observability + .as_ref() + .and_then(|observability| observability.sink_pid) + .into_iter() + .chain(receipt.bridge_proxies.iter().map(|proxy| proxy.pid)) + .chain( + receipt + .sites + .values() + .flat_map(|site| { + [ + Some(site.supervisor_pid), + site.process_pid, + site.port_forward_pid, + site.site_controller_pid, + ] + }) + .flatten(), + ) + .any(pid_is_alive) +} + +pub(crate) fn pick_free_port() -> u16 { + reserve_test_loopback_port() } pub(crate) fn docker_host_ip() -> String { @@ -331,14 +416,18 @@ pub(crate) fn http_request_with_timeout( .arg("--data") .arg(body); } - let output = command - .arg("-o") - .arg("-") - .arg("-w") - .arg("\n%{http_code}") - .arg(format!("http://127.0.0.1:{port}{path}")) - .output() - .ok()?; + let output = command_output_via_tempfiles( + command + .arg("-o") + .arg("-") + .arg("-w") + .arg("\n%{http_code}") + .arg(format!("http://127.0.0.1:{port}{path}")), + "curl http request", + ); + if !output.status.success() { + return None; + } let stdout = String::from_utf8_lossy(&output.stdout); let (body, status) = stdout.rsplit_once('\n')?; Some((status.trim().parse().ok()?, body.trim().to_string())) @@ -621,6 +710,7 @@ pub(crate) fn stop_child(child: &mut std::process::Child) { } pub(crate) fn amber_command() -> Command { + ensure_local_dev_image_tag_overrides(); let mut cmd = Command::new(env!("CARGO_BIN_EXE_amber")); cmd.env("AMBER_RUNTIME_BIN_DIR", runtime_bin_dir()); if let Some(kubeconfig) = env::var_os("AMBER_TEST_KIND_KUBECONFIG") { @@ -797,22 +887,22 @@ pub(crate) fn docker_supports_buildx() -> bool { pub(crate) fn ensure_amber_internal_images() { static READY: OnceLock<()> = OnceLock::new(); READY.get_or_init(|| { + ensure_local_dev_image_tag_overrides(); let root = workspace_root(); + let images = amber_internal_image_refs(); + ensure_docker_image(&images.router, &root.join("docker/amber-router/Dockerfile")); ensure_docker_image( - AMBER_ROUTER.reference, - &root.join("docker/amber-router/Dockerfile"), - ); - ensure_docker_image( - AMBER_PROVISIONER.reference, + &images.provisioner, &root.join("docker/amber-provisioner/Dockerfile"), ); + ensure_docker_image(&images.helper, &root.join("docker/amber-helper/Dockerfile")); ensure_docker_image( - AMBER_HELPER.reference, - &root.join("docker/amber-helper/Dockerfile"), + &images.site_controller, + &root.join("docker/amber-site-controller/Dockerfile"), ); ensure_docker_image( - AMBER_SITE_CONTROLLER.reference, - &root.join("docker/amber-site-controller/Dockerfile"), + &images.docker_gateway, + &root.join("docker/amber-docker-gateway/Dockerfile"), ); }); } @@ -825,6 +915,56 @@ pub(crate) fn ensure_internal_images() { }); } +#[derive(Clone)] +struct AmberInternalImageRefs { + router: String, + helper: String, + provisioner: String, + docker_gateway: String, + site_controller: String, +} + +fn amber_internal_image_refs() -> AmberInternalImageRefs { + let overrides = parse_dev_image_tag_overrides(INTERNAL_IMAGE_OVERRIDE_KEYS) + .unwrap_or_else(|err| panic!("invalid {DEV_IMAGE_TAGS_ENV}: {err}")); + let resolve = |image: &ImageRef, key: &str| { + overrides + .get(key) + .map(|tag| override_reference(image, tag)) + .unwrap_or_else(|| image.reference.to_string()) + }; + AmberInternalImageRefs { + router: resolve(&AMBER_ROUTER, "router"), + helper: resolve(&AMBER_HELPER, "helper"), + provisioner: resolve(&AMBER_PROVISIONER, "provisioner"), + docker_gateway: resolve(&AMBER_DOCKER_GATEWAY, "docker_gateway"), + site_controller: resolve(&AMBER_SITE_CONTROLLER, "site_controller"), + } +} + +fn ensure_local_dev_image_tag_overrides() { + static READY: OnceLock<()> = OnceLock::new(); + READY.get_or_init(|| { + if use_prebuilt_images() + || env::var_os("CI").is_some() + || env::var_os(DEV_IMAGE_TAGS_ENV).is_some() + { + return; + } + let mut hasher = DefaultHasher::new(); + workspace_root().hash(&mut hasher); + let tag = format!("dev-mixed-run-{:016x}", hasher.finish()); + let overrides = INTERNAL_IMAGE_OVERRIDE_KEYS + .iter() + .map(|key| format!("{key}={tag}")) + .collect::>() + .join(","); + unsafe { + env::set_var(DEV_IMAGE_TAGS_ENV, overrides); + } + }); +} + pub(crate) fn load_kind_image(cluster_name: &str, image: &str) { let mut last_error = None; for attempt in 1..=3 { @@ -944,23 +1084,27 @@ pub(crate) fn ensure_kind_internal_images(kind_cluster: &KindCluster) { ensure_internal_images(); static READY: OnceLock>> = OnceLock::new(); let name = kind_cluster.name.clone(); + let images = amber_internal_image_refs(); + let ready_key = format!( + "{name}|{}|{}|{}|{}|{}", + images.router, images.provisioner, images.helper, images.site_controller, TEST_APP_IMAGE + ); let loaded = READY.get_or_init(|| Mutex::new(BTreeSet::new())); { let loaded = loaded.lock().expect("kind image-load guard should lock"); - if loaded.contains(&name) { + if loaded.contains(&ready_key) { return; } } - load_kind_image(&name, AMBER_ROUTER.reference); - load_kind_image(&name, AMBER_PROVISIONER.reference); - load_kind_image(&name, AMBER_HELPER.reference); - load_kind_image(&name, AMBER_SITE_CONTROLLER.reference); - // The public test workload image is registry-pullable inside the kind cluster, so keep kind - // image preloading focused on the Amber images that only exist locally. + load_kind_image(&name, &images.router); + load_kind_image(&name, &images.provisioner); + load_kind_image(&name, &images.helper); + load_kind_image(&name, &images.site_controller); + load_kind_image(&name, TEST_APP_IMAGE); loaded .lock() .expect("kind image-load guard should lock") - .insert(name); + .insert(ready_key); } pub(crate) fn kill_pid(pid: u32) { @@ -970,6 +1114,338 @@ pub(crate) fn kill_pid(pid: u32) { } } +pub(crate) fn framework_child_artifact_dir( + run_root: &Path, + site_id: &str, + child_id: u64, +) -> PathBuf { + let manager_state = site_manager_state(run_root, site_id); + if manager_state["kind"].as_str() == Some("kubernetes") { + let artifact_dir = + framework_child_kubernetes_cache_artifact_dir(run_root, site_id, child_id); + materialize_kubernetes_child_artifact(&manager_state, child_id, &artifact_dir); + return artifact_dir; + } + + framework_child_host_artifact_dir(run_root, site_id, child_id) +} + +pub(crate) fn framework_control_state_snapshot(control_state_root: &Path) -> Value { + let mut live_children = Vec::new(); + let entries = fs::read_dir(control_state_root) + .unwrap_or_else(|err| panic!("failed to read {}: {err}", control_state_root.display())); + for entry in entries.filter_map(Result::ok) { + let Ok(file_type) = entry.file_type() else { + continue; + }; + if !file_type.is_dir() { + continue; + } + let Some(site_id) = entry.file_name().into_string().ok() else { + continue; + }; + let state = framework_site_control_state(control_state_root, &site_id); + if let Some(children) = state["live_children"].as_array() { + live_children.extend(children.iter().cloned()); + } + } + json!({ "live_children": live_children }) +} + +pub(crate) fn framework_child_is_absent(run_root: &Path, site_id: &str, child_id: u64) -> bool { + let manager_state = site_manager_state(run_root, site_id); + if manager_state["kind"].as_str() != Some("kubernetes") { + return !framework_child_host_child_root(run_root, site_id, child_id).exists(); + } + + if kubernetes_child_runtime_root_exists(&manager_state, child_id) { + return false; + } + + let cache_root = framework_child_kubernetes_cache_child_root(run_root, site_id, child_id); + if cache_root.exists() { + fs::remove_dir_all(&cache_root).unwrap_or_else(|err| { + panic!( + "failed to clear stale kubernetes child cache {}: {err}", + cache_root.display() + ) + }); + } + true +} + +fn framework_site_control_state(control_state_root: &Path, site_id: &str) -> Value { + let manager_state = site_manager_state_from_state_root(control_state_root, site_id); + let state_path = framework_site_control_state_path(control_state_root, site_id, &manager_state); + if manager_state["kind"].as_str() == Some("kubernetes") { + materialize_kubernetes_control_state(&manager_state, &state_path); + } + read_json(&state_path) +} + +fn site_manager_state(run_root: &Path, site_id: &str) -> Value { + site_manager_state_from_state_root(&run_root.join("state"), site_id) +} + +fn site_manager_state_from_state_root(state_root: &Path, site_id: &str) -> Value { + let manager_state_path = state_root.join(site_id).join("manager-state.json"); + serde_json::from_slice(&fs::read(&manager_state_path).unwrap_or_else(|err| { + panic!( + "failed to read site manager state {}: {err}", + manager_state_path.display() + ) + })) + .unwrap_or_else(|err| { + panic!( + "failed to parse site manager state {}: {err}", + manager_state_path.display() + ) + }) +} + +fn framework_site_control_state_path( + control_state_root: &Path, + site_id: &str, + manager_state: &Value, +) -> PathBuf { + if manager_state["kind"].as_str() == Some("kubernetes") { + return framework_kubernetes_cache_control_state_path(control_state_root, site_id); + } + control_state_root + .join(site_id) + .join("site-controller-state.json") +} + +fn framework_kubernetes_cache_control_state_path( + control_state_root: &Path, + site_id: &str, +) -> PathBuf { + control_state_root + .join(site_id) + .join("framework-component-kubernetes-cache") + .join("site-controller-state.json") +} + +fn framework_child_host_child_root(run_root: &Path, site_id: &str, child_id: u64) -> PathBuf { + run_root + .join("state") + .join(site_id) + .join("framework-component") + .join("children") + .join(child_id.to_string()) +} + +fn framework_child_host_artifact_dir(run_root: &Path, site_id: &str, child_id: u64) -> PathBuf { + framework_child_host_child_root(run_root, site_id, child_id).join("artifact") +} + +fn framework_child_kubernetes_cache_child_root( + run_root: &Path, + site_id: &str, + child_id: u64, +) -> PathBuf { + run_root + .join("state") + .join(site_id) + .join("framework-component-kubernetes-cache") + .join("children") + .join(child_id.to_string()) +} + +fn framework_child_kubernetes_cache_artifact_dir( + run_root: &Path, + site_id: &str, + child_id: u64, +) -> PathBuf { + framework_child_kubernetes_cache_child_root(run_root, site_id, child_id).join("artifact") +} + +fn kubernetes_child_runtime_root_exists(manager_state: &Value, child_id: u64) -> bool { + let namespace = manager_state["kubernetes_namespace"] + .as_str() + .unwrap_or_else(|| panic!("kubernetes manager state is missing kubernetes_namespace")); + let pod = kubernetes_site_controller_pod_name(manager_state, namespace); + let remote_child_root = format!("/amber/site/state/framework-component/children/{child_id}"); + let output = kubectl_for_manager_state(manager_state) + .arg("-n") + .arg(namespace) + .arg("exec") + .arg(&pod) + .arg("--") + .arg("sh") + .arg("-lc") + .arg(format!("test -d {remote_child_root}")) + .output() + .unwrap_or_else(|err| { + panic!( + "failed to probe kubernetes child runtime root {remote_child_root} in {pod}: {err}" + ) + }); + match output.status.code() { + Some(0) => true, + Some(1) => false, + _ => { + panic!( + "failed to probe kubernetes child runtime root {remote_child_root} in {pod}: {}", + String::from_utf8_lossy(&output.stderr) + ) + } + } +} + +fn materialize_kubernetes_child_artifact( + manager_state: &Value, + child_id: u64, + artifact_dir: &Path, +) { + if artifact_dir.is_dir() { + return; + } + let namespace = manager_state["kubernetes_namespace"] + .as_str() + .unwrap_or_else(|| panic!("kubernetes manager state is missing kubernetes_namespace")); + let pod = kubernetes_site_controller_pod_name(manager_state, namespace); + let remote_artifact_dir = + format!("/amber/site/state/framework-component/children/{child_id}/artifact"); + let parent = artifact_dir.parent().unwrap_or_else(|| { + panic!( + "artifact dir {} should have a parent", + artifact_dir.display() + ) + }); + fs::create_dir_all(parent).unwrap_or_else(|err| { + panic!( + "failed to create framework child parent {}: {err}", + parent.display() + ) + }); + let target = parent.join(format!("artifact-kubernetes-copy-{child_id}")); + if target.exists() { + fs::remove_dir_all(&target).unwrap_or_else(|err| { + panic!( + "failed to clear previous kubernetes child artifact copy {}: {err}", + target.display() + ) + }); + } + let mut command = kubectl_for_manager_state(manager_state); + let status = command + .arg("-n") + .arg(namespace) + .arg("cp") + .arg(format!("{pod}:{remote_artifact_dir}")) + .arg(&target) + .status() + .unwrap_or_else(|err| { + panic!( + "failed to copy kubernetes child artifact {remote_artifact_dir} from {pod}: {err}" + ) + }); + if !status.success() { + panic!( + "kubectl cp failed for kubernetes child artifact {remote_artifact_dir} from {pod} \ + with status {status}" + ); + } + fs::rename(&target, artifact_dir).unwrap_or_else(|err| { + panic!( + "failed to move kubernetes child artifact copy {} into {}: {err}", + target.display(), + artifact_dir.display() + ) + }); +} + +fn materialize_kubernetes_control_state(manager_state: &Value, state_path: &Path) { + let namespace = manager_state["kubernetes_namespace"] + .as_str() + .unwrap_or_else(|| panic!("kubernetes manager state is missing kubernetes_namespace")); + let pod = kubernetes_site_controller_pod_name(manager_state, namespace); + let parent = state_path.parent().unwrap_or_else(|| { + panic!( + "framework control state path {} should have a parent", + state_path.display() + ) + }); + fs::create_dir_all(parent).unwrap_or_else(|err| { + panic!( + "failed to create kubernetes control state cache parent {}: {err}", + parent.display() + ) + }); + let target = parent.join("site-controller-state-kubernetes-copy.json"); + let output = kubectl_for_manager_state(manager_state) + .arg("-n") + .arg(namespace) + .arg("exec") + .arg(&pod) + .arg("--") + .arg("cat") + .arg("/amber/site/state/site-controller-state.json") + .output() + .unwrap_or_else(|err| { + panic!("failed to read kubernetes control state from {pod} in {namespace}: {err}") + }); + if !output.status.success() { + panic!( + "failed to read kubernetes control state from {pod} in {namespace}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + fs::write(&target, &output.stdout).unwrap_or_else(|err| { + panic!( + "failed to write temporary kubernetes control state cache {}: {err}", + target.display() + ) + }); + fs::rename(&target, state_path).unwrap_or_else(|err| { + panic!( + "failed to move kubernetes control state cache {} into {}: {err}", + target.display(), + state_path.display() + ) + }); +} + +fn kubernetes_site_controller_pod_name(manager_state: &Value, namespace: &str) -> String { + let mut command = kubectl_for_manager_state(manager_state); + let output = command + .arg("-n") + .arg(namespace) + .arg("get") + .arg("pods") + .arg("-l") + .arg("amber.io/component=amber-site-controller") + .arg("-o") + .arg("jsonpath={.items[0].metadata.name}") + .output() + .unwrap_or_else(|err| { + panic!("failed to query kubernetes site-controller pod in {namespace}: {err}") + }); + if !output.status.success() { + panic!( + "failed to query kubernetes site-controller pod in {namespace}: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + let pod = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if pod.is_empty() { + panic!("kubernetes site-controller pod is missing in namespace {namespace}"); + } + pod +} + +fn kubectl_for_manager_state(manager_state: &Value) -> Command { + let mut command = Command::new("kubectl"); + if let Some(kubeconfig) = env::var_os("AMBER_TEST_KIND_KUBECONFIG") { + command.env("KUBECONFIG", kubeconfig); + } + if let Some(context) = manager_state["context"].as_str() { + command.arg("--context").arg(context); + } + command +} + pub(crate) fn pid_is_alive(pid: u32) -> bool { #[cfg(unix)] { @@ -1035,14 +1511,15 @@ impl RunHandle { if self.stopped { return; } - let output = amber_command() - .arg("stop") - .arg(&self.run_id) - .arg("--storage-root") - .arg(&self.storage_root) - .envs(self.command_env.iter().map(|(key, value)| (key, value))) - .output() - .expect("failed to run amber stop"); + let output = command_output_via_tempfiles( + amber_command() + .arg("stop") + .arg(&self.run_id) + .arg("--storage-root") + .arg(&self.storage_root) + .envs(self.command_env.iter().map(|(key, value)| (key, value))), + "amber stop", + ); assert!( output.status.success(), "amber stop failed\nstdout:\n{}\nstderr:\n{}", @@ -1058,17 +1535,47 @@ impl Drop for RunHandle { if self.stopped { return; } - let _ = amber_command() - .arg("stop") - .arg(&self.run_id) - .arg("--storage-root") - .arg(&self.storage_root) - .envs(self.command_env.iter().map(|(key, value)| (key, value))) - .output(); + let _ = command_output_via_tempfiles( + amber_command() + .arg("stop") + .arg(&self.run_id) + .arg("--storage-root") + .arg(&self.storage_root) + .envs(self.command_env.iter().map(|(key, value)| (key, value))), + "amber stop", + ); self.stopped = true; } } +pub(crate) fn command_output_via_tempfiles( + command: &mut Command, + label: &str, +) -> std::process::Output { + let stdout_file = tempfile::NamedTempFile::new() + .unwrap_or_else(|err| panic!("failed to create stdout temp file for {label}: {err}")); + let stderr_file = tempfile::NamedTempFile::new() + .unwrap_or_else(|err| panic!("failed to create stderr temp file for {label}: {err}")); + let status = command + .stdout(Stdio::from(stdout_file.reopen().unwrap_or_else(|err| { + panic!("failed to reopen stdout temp file for {label}: {err}") + }))) + .stderr(Stdio::from(stderr_file.reopen().unwrap_or_else(|err| { + panic!("failed to reopen stderr temp file for {label}: {err}") + }))) + .status() + .unwrap_or_else(|err| panic!("failed to run {label}: {err}")); + let stdout = fs::read(stdout_file.path()) + .unwrap_or_else(|err| panic!("failed to read stdout temp file for {label}: {err}")); + let stderr = fs::read(stderr_file.path()) + .unwrap_or_else(|err| panic!("failed to read stderr temp file for {label}: {err}")); + std::process::Output { + status, + stdout, + stderr, + } +} + pub(crate) fn parse_run_id(stdout: &[u8]) -> String { String::from_utf8_lossy(stdout) .lines() @@ -1099,17 +1606,18 @@ pub(crate) fn run_manifest_with_args_and_env( extra_args: &[&str], extra_env: &[(&str, &str)], ) -> RunHandle { - let output = amber_command() - .arg("run") - .arg(manifest) - .arg("--placement") - .arg(placement) - .arg("--storage-root") - .arg(storage_root) - .args(extra_args) - .envs(extra_env.iter().copied()) - .output() - .expect("failed to run amber run"); + let output = command_output_via_tempfiles( + amber_command() + .arg("run") + .arg(manifest) + .arg("--placement") + .arg(placement) + .arg("--storage-root") + .arg(storage_root) + .args(extra_args) + .envs(extra_env.iter().copied()), + "amber run", + ); assert!( output.status.success(), "amber run failed\nstdout:\n{}\nstderr:\n{}{}", @@ -1189,7 +1697,7 @@ pub(crate) fn run_manifest_expect_failure_with_env( .arg(storage_root) .args(extra_args) .envs(extra_env.iter().copied()); - let output = cmd.output().expect("failed to run amber run"); + let output = command_output_via_tempfiles(&mut cmd, "amber run"); assert!( !output.status.success(), "amber run unexpectedly succeeded\nstdout:\n{}\nstderr:\n{}", @@ -1209,21 +1717,22 @@ pub(crate) fn dry_run_manifest( bundle_root: &Path, extra_args: &[&str], ) -> std::process::Output { - amber_command() - .arg("run") - .arg("-Z") - .arg("unstable-options") - .arg(manifest) - .arg("--placement") - .arg(placement) - .arg("--storage-root") - .arg(storage_root) - .args(extra_args) - .arg("--dry-run") - .arg("--emit-launch-bundle") - .arg(bundle_root) - .output() - .expect("failed to run amber run --dry-run") + command_output_via_tempfiles( + amber_command() + .arg("run") + .arg("-Z") + .arg("unstable-options") + .arg(manifest) + .arg("--placement") + .arg(placement) + .arg("--storage-root") + .arg(storage_root) + .args(extra_args) + .arg("--dry-run") + .arg("--emit-launch-bundle") + .arg(bundle_root), + "amber run --dry-run", + ) } pub(crate) fn spawn_run_manifest_with_env( @@ -1623,6 +2132,8 @@ pub(crate) fn serve_host_http_request(stream: &mut TcpStream) { pub(crate) struct KindClusterGuard { name: String, kubeconfig: PathBuf, + previous_cluster_name: Option, + previous_kubeconfig: Option, } impl KindClusterGuard { @@ -1655,9 +2166,17 @@ impl KindClusterGuard { .status(); panic!("kind create cluster failed with status {status}"); } + let previous_cluster_name = env::var_os("AMBER_TEST_KIND_CLUSTER_NAME"); + let previous_kubeconfig = env::var_os("AMBER_TEST_KIND_KUBECONFIG"); + unsafe { + env::set_var("AMBER_TEST_KIND_CLUSTER_NAME", &name); + env::set_var("AMBER_TEST_KIND_KUBECONFIG", kubeconfig); + } Self { name, kubeconfig: kubeconfig.to_path_buf(), + previous_cluster_name, + previous_kubeconfig, } } } @@ -1672,6 +2191,16 @@ impl Drop for KindClusterGuard { .arg("--kubeconfig") .arg(&self.kubeconfig) .status(); + unsafe { + match &self.previous_cluster_name { + Some(value) => env::set_var("AMBER_TEST_KIND_CLUSTER_NAME", value), + None => env::remove_var("AMBER_TEST_KIND_CLUSTER_NAME"), + } + match &self.previous_kubeconfig { + Some(value) => env::set_var("AMBER_TEST_KIND_KUBECONFIG", value), + None => env::remove_var("AMBER_TEST_KIND_KUBECONFIG"), + } + } } } @@ -2282,6 +2811,21 @@ pub(crate) fn namespace_exists(namespace: &str, kubeconfig: &Path, context: &str mod tests { use super::*; + #[test] + fn command_output_via_tempfiles_does_not_wait_for_background_stdout_writer() { + let started = Instant::now(); + let output = command_output_via_tempfiles( + Command::new("sh").arg("-c").arg("(sleep 2) & printf done"), + "background stdout writer", + ); + assert!(output.status.success()); + assert_eq!(String::from_utf8_lossy(&output.stdout), "done"); + assert!( + started.elapsed() < Duration::from_secs(1), + "capturing to temp files should wait for the command, not background writers" + ); + } + #[test] fn source_is_newer_than_image_detects_newer_sources() { let image_created = UNIX_EPOCH + Duration::from_secs(10); @@ -2298,4 +2842,67 @@ mod tests { )); assert!(!source_is_newer_than_image(image_created, None)); } + + #[test] + fn framework_child_artifact_dir_uses_dedicated_kubernetes_cache() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let manager_state_path = temp + .path() + .join("state") + .join("kind_local") + .join("manager-state.json"); + fs::create_dir_all(manager_state_path.parent().expect("manager state parent")) + .expect("manager state dir should create"); + fs::write( + &manager_state_path, + serde_json::to_vec(&json!({ + "kind": "kubernetes", + "kubernetes_namespace": "amber-test", + })) + .expect("manager state should serialize"), + ) + .expect("manager state should write"); + + let cache_dir = framework_child_kubernetes_cache_artifact_dir(temp.path(), "kind_local", 7); + fs::create_dir_all(&cache_dir).expect("kubernetes cache dir should exist"); + + let artifact_dir = framework_child_artifact_dir(temp.path(), "kind_local", 7); + assert_eq!(artifact_dir, cache_dir); + assert!( + !artifact_dir.starts_with( + temp.path() + .join("state") + .join("kind_local") + .join("framework-component") + .join("children") + ), + "kubernetes child artifact inspection should not reuse the authoritative host state \ + path" + ); + } + + #[test] + fn framework_control_state_path_uses_dedicated_kubernetes_cache() { + let state_root = Path::new("/tmp/amber-run/state"); + let manager_state = json!({ + "kind": "kubernetes", + "kubernetes_namespace": "amber-test", + }); + + let state_path = + framework_site_control_state_path(state_root, "kind_local", &manager_state); + assert_eq!( + state_path, + framework_kubernetes_cache_control_state_path(state_root, "kind_local") + ); + assert!( + !state_path.ends_with("site-controller-state.json") + || state_path + .parent() + .and_then(Path::file_name) + .is_some_and(|name| name == "framework-component-kubernetes-cache"), + "kubernetes framework control state should use the dedicated cache path, got {}", + state_path.display() + ); + } } diff --git a/cli/tests/test_support/port_allocator.rs b/cli/tests/test_support/port_allocator.rs new file mode 100644 index 00000000..ea6a60b4 --- /dev/null +++ b/cli/tests/test_support/port_allocator.rs @@ -0,0 +1,63 @@ +use std::{ + collections::BTreeSet, + net::{SocketAddr, TcpListener}, + sync::{Mutex, OnceLock}, +}; + +const TEST_PORT_RANGE_START: u16 = 20_000; +const TEST_PORT_RANGE_END: u16 = 30_000; +static RESERVED_TEST_PORTS: OnceLock>> = OnceLock::new(); + +pub(crate) fn reserve_test_loopback_port() -> u16 { + let reserved = RESERVED_TEST_PORTS.get_or_init(|| Mutex::new(BTreeSet::new())); + let mut reserved = reserved + .lock() + .expect("test port allocator should not be poisoned"); + let span = u32::from(TEST_PORT_RANGE_END - TEST_PORT_RANGE_START); + let mut next = + TEST_PORT_RANGE_START + (std::process::id() % span) as u16 + reserved.len() as u16; + for _ in 0..usize::from(TEST_PORT_RANGE_END - TEST_PORT_RANGE_START) { + if next >= TEST_PORT_RANGE_END { + next = TEST_PORT_RANGE_START; + } + let port = next; + next += 1; + if reserved.contains(&port) { + continue; + } + match TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], port))) { + Ok(listener) => { + drop(listener); + reserved.insert(port); + return port; + } + Err(_) => continue, + } + } + panic!( + "failed to allocate a unique test loopback port in {}-{}", + TEST_PORT_RANGE_START, + TEST_PORT_RANGE_END - 1 + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reserve_test_loopback_port_uses_reserved_test_range() { + let port = reserve_test_loopback_port(); + assert!( + (TEST_PORT_RANGE_START..TEST_PORT_RANGE_END).contains(&port), + "test allocator returned port outside reserved range: {port}" + ); + } + + #[test] + fn reserve_test_loopback_port_does_not_reuse_ports_in_process() { + let first = reserve_test_loopback_port(); + let second = reserve_test_loopback_port(); + assert_ne!(first, second); + } +} diff --git a/cli/tests/vm_smoke.rs b/cli/tests/vm_smoke.rs index e567272a..b46bc685 100644 --- a/cli/tests/vm_smoke.rs +++ b/cli/tests/vm_smoke.rs @@ -2,6 +2,8 @@ #[path = "test_support/cloud_image.rs"] mod cloud_image_support; +#[path = "test_support/port_allocator.rs"] +mod port_allocator_support; #[path = "test_support/target_dir.rs"] mod target_dir_support; #[path = "test_support/workspace_root.rs"] @@ -10,7 +12,7 @@ mod workspace_root_support; use std::{ env, fs, io::{Read, Write}, - net::{SocketAddr, TcpListener, TcpStream}, + net::{SocketAddr, TcpStream}, path::{Path, PathBuf}, process::{Command, Stdio}, thread, @@ -18,6 +20,7 @@ use std::{ }; use cloud_image_support::default_host_arch_cloud_image_filename; +use port_allocator_support::reserve_test_loopback_port; use target_dir_support::cargo_target_dir; use workspace_root_support::workspace_root; @@ -86,8 +89,7 @@ fn emit_wait_heartbeat( } fn pick_free_port() -> u16 { - let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))).unwrap(); - listener.local_addr().unwrap().port() + reserve_test_loopback_port() } fn ensure_runtime_binaries_built(workspace_root: &Path) -> PathBuf { diff --git a/compiler/src/targets/mesh/docker_compose/mod.rs b/compiler/src/targets/mesh/docker_compose/mod.rs index 2be9368c..8e2c398b 100644 --- a/compiler/src/targets/mesh/docker_compose/mod.rs +++ b/compiler/src/targets/mesh/docker_compose/mod.rs @@ -791,6 +791,9 @@ fn render_docker_compose_inner( ); push_router_observability_env(&mut sidecar_env_entries); sidecar_service.environment = Some(Environment::List(sidecar_env_entries)); + sidecar_service + .extra_hosts + .push(HOST_GATEWAY_ENTRY.to_string()); sidecar_service .networks .insert(MESH_NETWORK_NAME.to_string(), EmptyMap::default()); diff --git a/compiler/src/targets/mesh/docker_compose/tests/rendering.rs b/compiler/src/targets/mesh/docker_compose/tests/rendering.rs index 5dae1ce5..cc068d4f 100644 --- a/compiler/src/targets/mesh/docker_compose/tests/rendering.rs +++ b/compiler/src/targets/mesh/docker_compose/tests/rendering.rs @@ -347,6 +347,13 @@ fn compose_emits_otelcol_agent_and_wires_router_otel_env() { .contains_key("amber_egress_c0-component-net"), "{yaml}" ); + assert!( + sidecar + .extra_hosts + .iter() + .any(|entry| entry == super::HOST_GATEWAY_ENTRY), + "{yaml}" + ); assert!( !sidecar.networks.contains_key(super::BOUNDARY_NETWORK_NAME), "{yaml}" diff --git a/runtime/proxy/src/lib.rs b/runtime/proxy/src/lib.rs index c2dad81d..82731dcd 100644 --- a/runtime/proxy/src/lib.rs +++ b/runtime/proxy/src/lib.rs @@ -1993,13 +1993,44 @@ where .map_err(|_| ControlUpdateError::Retryable)? .map_err(|_| ControlUpdateError::Retryable)?; let mut buf = Vec::new(); - tokio::time::timeout(CONTROL_REQUEST_TIMEOUT, stream.read_to_end(&mut buf)) - .await - .map_err(|_| ControlUpdateError::Retryable)? - .map_err(|_| ControlUpdateError::Retryable)?; + let mut chunk = [0u8; 8192]; + loop { + let read = tokio::time::timeout(CONTROL_REQUEST_TIMEOUT, stream.read(&mut chunk)) + .await + .map_err(|_| ControlUpdateError::Retryable)? + .map_err(|_| ControlUpdateError::Retryable)?; + if read == 0 { + break; + } + buf.extend_from_slice(&chunk[..read]); + if let Some(response_len) = http_response_length(&buf) + && buf.len() >= response_len + { + break; + } + } Ok(String::from_utf8_lossy(&buf).to_string()) } +fn http_response_length(buf: &[u8]) -> Option { + let header_end = find_header_end(buf)?; + let header = std::str::from_utf8(&buf[..header_end]).ok()?; + let content_length = header.lines().find_map(|line| { + let (name, value) = line.split_once(':')?; + name.trim() + .eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok()) + .flatten() + })?; + Some(header_end + content_length) +} + +fn find_header_end(buf: &[u8]) -> Option { + buf.windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|index| index + 4) +} + fn mesh_protocol_from_metadata(protocol: &str) -> Result { Ok(match protocol { "http" | "https" => MeshProtocol::Http, diff --git a/runtime/proxy/src/tests.rs b/runtime/proxy/src/tests.rs index af9736b0..68f74f9c 100644 --- a/runtime/proxy/src/tests.rs +++ b/runtime/proxy/src/tests.rs @@ -637,3 +637,45 @@ async fn try_fetch_router_identity_times_out_stalled_control_requests() { ); handle.join().expect("listener thread should finish"); } + +#[tokio::test] +async fn try_fetch_router_identity_accepts_complete_http_response_without_eof() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + let listener = tokio::net::TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) + .await + .expect("listener should bind"); + let addr = listener.local_addr().expect("listener addr"); + let public_key = base64::engine::general_purpose::STANDARD.encode([7u8; 32]); + let body = format!( + r#"{{"id":"/site/test/router","public_key":"{}","mesh_scope":"scope"}}"#, + public_key + ); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("listener should accept"); + let mut request = [0u8; 256]; + let _ = stream + .read(&mut request) + .await + .expect("request should be readable"); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}", + body.len(), + body + ); + stream + .write_all(response.as_bytes()) + .await + .expect("response should be writable"); + tokio::time::sleep(CONTROL_REQUEST_TIMEOUT + Duration::from_secs(1)).await; + }); + + let identity = match try_fetch_router_identity(&ControlEndpoint::Tcp(addr.to_string())).await { + Ok(identity) => identity, + Err(_) => panic!("complete HTTP response should not require EOF"), + }; + assert_eq!(identity.id, "/site/test/router"); + assert_eq!(identity.public_key, [7u8; 32]); + assert_eq!(identity.mesh_scope.as_deref(), Some("scope")); + server.await.expect("server should finish"); +} diff --git a/runtime/router/src/dynamic_caps.rs b/runtime/router/src/dynamic_caps.rs index 1b6dcf08..d31cafd4 100644 --- a/runtime/router/src/dynamic_caps.rs +++ b/runtime/router/src/dynamic_caps.rs @@ -130,7 +130,6 @@ struct DynamicHandleRecord { handle_id: String, held_id: String, descriptor: DescriptorIr, - upstream: Option, } #[derive(Clone)] @@ -145,6 +144,49 @@ struct DynamicHandleUpstreamTasks { bridge_task: tokio::task::JoinHandle>, } +// Dynamic handle upstreams are established per request, so the response body +// must retain the upstream connection until streaming completes. +struct DynamicHandleResponseBody { + inner: BoxBody, + _upstream: DynamicHandleUpstream, +} + +impl hyper::body::Body for DynamicHandleResponseBody { + type Data = Bytes; + type Error = hyper::Error; + + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + let this = self.get_mut(); + Pin::new(&mut this.inner).poll_frame(cx) + } + + fn is_end_stream(&self) -> bool { + self.inner.is_end_stream() + } + + fn size_hint(&self) -> hyper::body::SizeHint { + self.inner.size_hint() + } +} + +fn retain_dynamic_handle_upstream( + response: Response, + upstream: DynamicHandleUpstream, +) -> Response { + let (parts, body) = response.into_parts(); + Response::from_parts( + parts, + DynamicHandleResponseBody { + inner: body, + _upstream: upstream, + } + .boxed(), + ) +} + struct DynamicHandleProxyRoute { handle_id: String, forwarded_path: String, @@ -162,12 +204,6 @@ struct HeldWatchSnapshot { materializations: BTreeMap>, } -impl DynamicHandleUpstream { - fn is_alive(&self) -> bool { - !self._tasks.conn_task.is_finished() && !self._tasks.bridge_task.is_finished() - } -} - impl Drop for DynamicHandleUpstreamTasks { fn drop(&mut self) { self.conn_task.abort(); @@ -623,7 +659,7 @@ impl DynamicCapsRuntime { source: DynamicCapabilityControlSourceRequest, descriptor_hint: Option, ) -> Result { - let resolved = self.control_resolve_origin(source.clone()).await?; + let resolved = self.control_resolve_origin(source).await?; if resolved.origin_protocol != "http" { return Err(self.protocol_error( ProtocolErrorCode::PathEstablishmentFailed, @@ -661,13 +697,10 @@ impl DynamicCapsRuntime { handle_id: handle_id.clone(), held_id: resolved.held_id.clone(), descriptor: descriptor_hint.unwrap_or_else(|| resolved.descriptor.clone()), - upstream: None, }, ); handle_id }; - self.refresh_dynamic_handle_upstream(&handle_id, &resolved) - .await?; Ok(MaterializeResponse { held_id: resolved.held_id, handle_id: handle_id.clone(), @@ -675,11 +708,10 @@ impl DynamicCapsRuntime { }) } - async fn refresh_dynamic_handle_upstream( + async fn establish_dynamic_handle_upstream( &self, - handle_id: &str, resolved: &ControlDynamicResolveOriginResponse, - ) -> Result<(), ProtocolErrorResponse> { + ) -> Result { let peer_key = base64::engine::general_purpose::STANDARD .decode(resolved.origin_peer_key_b64.as_bytes()) .map_err(|err| { @@ -742,30 +774,21 @@ impl DynamicCapsRuntime { ); } }); - let upstream = DynamicHandleUpstream { + Ok(DynamicHandleUpstream { route_id: Arc::::from(resolved.origin_route_id.as_str()), sender: Arc::new(Mutex::new(sender)), _tasks: Arc::new(DynamicHandleUpstreamTasks { conn_task, bridge_task, }), - }; - let mut state = self.handles.lock().await; - let record = state.handles_by_id.get_mut(handle_id).ok_or_else(|| { - self.protocol_error( - ProtocolErrorCode::UnknownHandle, - "dynamic capability handle disappeared during materialization", - ) - })?; - record.upstream = Some(upstream); - Ok(()) + }) } - async fn ensure_dynamic_handle_upstream( + async fn resolve_dynamic_handle_request( &self, handle_id: &str, ) -> Result<(DynamicHandleRecordView, DynamicHandleUpstream), ProtocolErrorResponse> { - let (held_id, descriptor, current_upstream) = { + let (held_id, descriptor) = { let state = self.handles.lock().await; let record = state.handles_by_id.get(handle_id).ok_or_else(|| { self.protocol_error( @@ -773,11 +796,7 @@ impl DynamicCapsRuntime { "dynamic capability handle does not exist", ) })?; - ( - record.held_id.clone(), - record.descriptor.clone(), - record.upstream.clone(), - ) + (record.held_id.clone(), record.descriptor.clone()) }; let detail = self.control_held_detail(&held_id).await?; if detail.summary.state != HeldEntryState::Live { @@ -786,39 +805,15 @@ impl DynamicCapsRuntime { "dynamic capability handle has been revoked", )); } - if let Some(upstream) = current_upstream - && upstream.is_alive() - { - return Ok(( - DynamicHandleRecordView { - held_id, - descriptor, - }, - upstream, - )); - } let source = self.source_from_held_detail(&detail)?; let resolved = self.control_resolve_origin(source).await?; - self.refresh_dynamic_handle_upstream(handle_id, &resolved) - .await?; - let state = self.handles.lock().await; - let record = state.handles_by_id.get(handle_id).ok_or_else(|| { - self.protocol_error( - ProtocolErrorCode::UnknownHandle, - "dynamic capability handle disappeared during refresh", - ) - })?; + let upstream = self.establish_dynamic_handle_upstream(&resolved).await?; Ok(( DynamicHandleRecordView { - held_id: record.held_id.clone(), - descriptor: record.descriptor.clone(), + held_id, + descriptor, }, - record.upstream.clone().ok_or_else(|| { - self.protocol_error( - ProtocolErrorCode::PathEstablishmentFailed, - "dynamic capability upstream was not established", - ) - })?, + upstream, )) } @@ -1043,7 +1038,7 @@ impl DynamicCapsRuntime { route: DynamicHandleProxyRoute, req: Request, ) -> Response { - let Ok((record, upstream)) = self.ensure_dynamic_handle_upstream(&route.handle_id).await + let Ok((record, upstream)) = self.resolve_dynamic_handle_request(&route.handle_id).await else { return error_response( StatusCode::GONE, @@ -1068,35 +1063,42 @@ impl DynamicCapsRuntime { } } let req = Request::from_parts(parts.0, parts.1); - proxy_outbound_http_request( - OutboundHttpProxyState { - upstream: upstream.sender.clone(), - plugins: if record.descriptor.kind == "a2a" { - Arc::from([Arc::new(a2a::A2aUrlRewritePlugin::new( - self.a2a_url_rewrite_table.clone(), - )) as Arc]) - } else { - Arc::from(Vec::>::new()) - }, - route_id: upstream.route_id.clone(), - peer_id: Arc::::from(self.config.identity.id.as_str()), - labels: HttpExchangeLabels { - kind: HttpEdgeKind::Binding, - emit_telemetry: true, - slot: None, - capability: Arc::::from(record.descriptor.label.as_str()), - capability_kind: Some(Arc::::from(record.descriptor.kind.as_str())), - capability_profile: record.descriptor.profile.as_deref().map(Arc::::from), - source_component: Some(Arc::::from(self.config.identity.id.as_str())), - source_endpoint: Arc::::from(record.held_id.as_str()), - destination_component: None, - destination_endpoint: Arc::::from(record.descriptor.label.as_str()), + retain_dynamic_handle_upstream( + proxy_outbound_http_request( + OutboundHttpProxyState { + upstream: upstream.sender.clone(), + plugins: if record.descriptor.kind == "a2a" { + Arc::from([Arc::new(a2a::A2aUrlRewritePlugin::new( + self.a2a_url_rewrite_table.clone(), + )) as Arc]) + } else { + Arc::from(Vec::>::new()) + }, + route_id: upstream.route_id.clone(), + peer_id: Arc::::from(self.config.identity.id.as_str()), + labels: HttpExchangeLabels { + kind: HttpEdgeKind::Binding, + emit_telemetry: true, + slot: None, + capability: Arc::::from(record.descriptor.label.as_str()), + capability_kind: Some(Arc::::from(record.descriptor.kind.as_str())), + capability_profile: record + .descriptor + .profile + .as_deref() + .map(Arc::::from), + source_component: Some(Arc::::from(self.config.identity.id.as_str())), + source_endpoint: Arc::::from(record.held_id.as_str()), + destination_component: None, + destination_endpoint: Arc::::from(record.descriptor.label.as_str()), + }, + dynamic_caps: Some(Arc::new(self.clone())), }, - dynamic_caps: Some(Arc::new(self.clone())), - }, - req, + req, + ) + .await, + upstream, ) - .await } } diff --git a/runtime/router/src/tests.rs b/runtime/router/src/tests.rs index cbdaa459..77d7b019 100644 --- a/runtime/router/src/tests.rs +++ b/runtime/router/src/tests.rs @@ -2281,6 +2281,215 @@ async fn outbound_http_proxy_overwrites_internal_framework_auth_headers() { .expect("upstream connection task should join"); } +#[tokio::test] +async fn outbound_http_proxy_keeps_ephemeral_upstream_alive_for_response_body() { + let (upstream_client, upstream_server) = duplex(64 * 1024); + let upstream_task = tokio::spawn(async move { + let service = service_fn(|_req: Request| async move { + Ok::<_, std::convert::Infallible>( + Response::builder() + .status(StatusCode::OK) + .body( + Full::new(Bytes::from_static(b"a")) + .map_err(|never| match never {}) + .boxed(), + ) + .expect("upstream response should build"), + ) + }); + http1::Builder::new() + .serve_connection(TokioIo::new(upstream_server), service) + .await + .expect("upstream server should complete"); + }); + + let (sender, conn) = client_http1::handshake(TokioIo::new(upstream_client)) + .await + .expect("upstream handshake should succeed"); + let upstream_conn_task = tokio::spawn(async move { + conn.await.expect("upstream connection should complete"); + }); + + let state_slot = Arc::new(Mutex::new(Some(OutboundHttpProxyState { + upstream: Arc::new(Mutex::new(sender)), + plugins: Arc::from(Vec::>::new()), + route_id: Arc::::from("dynamic-handle-route"), + peer_id: Arc::::from("/component/dynamic"), + labels: test_http_exchange_labels(), + dynamic_caps: None, + }))); + + let (proxy_client, proxy_server) = duplex(64 * 1024); + let proxy_task = tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let state_slot = state_slot.clone(); + async move { + let state = state_slot + .lock() + .await + .take() + .expect("proxy should only serve one request"); + Ok::<_, std::convert::Infallible>(proxy_outbound_http_request(state, req).await) + } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(proxy_server), service) + .await + .expect("proxy server should complete"); + }); + + let (mut client, conn) = client_http1::handshake(TokioIo::new(proxy_client)) + .await + .expect("proxy handshake should succeed"); + let proxy_conn_task = tokio::spawn(async move { + conn.await.expect("proxy connection should complete"); + }); + + let response = client + .send_request( + Request::builder() + .uri("/id") + .body(empty_box_body()) + .expect("request should build"), + ) + .await + .expect("proxy request should succeed"); + assert_eq!(response.status(), StatusCode::OK); + let body = response + .into_body() + .collect() + .await + .expect("response body should collect") + .to_bytes(); + assert_eq!(body.as_ref(), b"a"); + drop(client); + + proxy_task.await.expect("proxy task should join"); + proxy_conn_task + .await + .expect("proxy connection task should join"); + upstream_task.await.expect("upstream task should join"); + upstream_conn_task + .await + .expect("upstream connection task should join"); +} + +#[tokio::test] +async fn proxy_noise_to_plain_preserves_connection_close_response_body() { + let client_config = test_mesh_config(); + let server_identity = MeshIdentity::generate("provider-router", Some("test-scope".to_string())); + let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) + .await + .expect("mesh listener should bind"); + let addr = listener.local_addr().expect("mesh listener addr"); + let server_config = MeshConfig { + identity: server_identity.clone(), + peers: vec![MeshPeer { + id: client_config.identity.id.clone(), + public_key: client_config.identity.public_key, + }], + ..test_mesh_config() + }; + let trust = Arc::new(TrustBundle::new(&server_config).expect("mesh server trust")); + let noise_keys = noise_keys_for_identity(&server_config.identity).expect("noise keys"); + + let server_task = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("mesh peer should accept"); + let mut session = accept_noise(stream, &noise_keys, trust.as_ref()) + .await + .expect("mesh peer should accept noise"); + let open = session.recv_open().await.expect("open frame"); + assert_eq!(open.capability, "provider.api"); + + let (local, remote) = duplex(64 * 1024); + let bridge = tokio::spawn(async move { proxy_noise_to_plain(&mut session, local).await }); + let service = service_fn(|req: Request| async move { + assert_eq!(req.uri().path(), "/id"); + assert_eq!( + req.headers() + .get(header::CONNECTION) + .and_then(|value| value.to_str().ok()), + Some("close"), + ); + Ok::<_, std::convert::Infallible>( + Response::builder() + .status(StatusCode::OK) + .body( + Full::new(Bytes::from_static(b"provider")) + .map_err(|never| match never {}) + .boxed(), + ) + .expect("mesh response should build"), + ) + }); + http1::Builder::new() + .serve_connection(TokioIo::new(remote), service) + .await + .expect("mesh http server should complete"); + bridge + .await + .expect("mesh bridge task should complete") + .expect("mesh bridge should succeed"); + }); + + let mut outbound = connect_noise_with_key( + &addr.to_string(), + &server_identity.id, + server_identity.public_key, + &client_config, + ) + .await + .expect("mesh client should connect"); + outbound + .send_open(&OpenFrame { + route_id: "dynamic-handle-route".to_string(), + capability: "provider.api".to_string(), + protocol: MeshProtocol::Http, + slot: None, + capability_kind: None, + capability_profile: None, + }) + .await + .expect("open frame should send"); + + let (local, remote) = duplex(64 * 1024); + let client_bridge = + tokio::spawn(async move { proxy_noise_to_plain(&mut outbound, remote).await }); + let (mut sender, conn) = client_http1::handshake(TokioIo::new(local)) + .await + .expect("client handshake should succeed"); + let conn_task = tokio::spawn(async move { + conn.await.expect("client connection should complete"); + }); + + let response = sender + .send_request( + Request::builder() + .uri("/id") + .header(header::CONNECTION, "close") + .body(empty_box_body()) + .expect("request should build"), + ) + .await + .expect("response should arrive"); + assert_eq!(response.status(), StatusCode::OK); + let body = response + .into_body() + .collect() + .await + .expect("response body should collect") + .to_bytes(); + assert_eq!(body.as_ref(), b"provider"); + drop(sender); + + conn_task.await.expect("client connection task should join"); + client_bridge + .await + .expect("client bridge task should complete") + .expect("client bridge should succeed"); + server_task.await.expect("server task should join"); +} + #[tokio::test] async fn framework_external_proxy_preserves_forwarded_internal_auth_headers() { let captured = Arc::new(StdMutex::new(None::<(String, String)>)); diff --git a/runtime/site-controller/src/control_state_api.rs b/runtime/site-controller/src/control_state_api.rs index cd3b174a..5c469f5c 100644 --- a/runtime/site-controller/src/control_state_api.rs +++ b/runtime/site-controller/src/control_state_api.rs @@ -227,39 +227,48 @@ pub(crate) async fn resolve_dynamic_capability_origin_internal( )) })?; let _origin_runtime = local_component_runtime(app, &state, &root.holder_component_id)?; - let origin_manager_state = load_site_manager_state(app, &app.controller_plan.site_id)?; let origin_peer = local_router_identity_for_overlay(app).await?; - let origin_peer_addr = origin_manager_state - .router_mesh_addr - .as_deref() - .ok_or_else(|| { - ProtocolApiError::from(protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "site `{}` does not expose a live router mesh address for dynamic capability \ - publication", - app.controller_plan.site_id - ), - )) - }) - .and_then(|router_mesh_addr| { - app.runtime - .router_mesh_addr_for_consumer( - app.controller_plan.kind, - request.holder_site_kind, - router_mesh_addr, - ) - .map_err(|err| { - ProtocolApiError::from(protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "site `{}` exposes an invalid live router mesh address for dynamic \ - capability publication: {err}", - app.controller_plan.site_id - ), - )) - }) - })?; + let origin_state = load_site_manager_state(app, &app.controller_plan.site_id)?; + let origin_receipt = super::orchestration::site_receipt_from_manager_state(&origin_state); + let origin_peer_addr = if let Some(router_mesh_addr) = + crate::runtime_api::published_router_mesh_addr_for_consumer_kind( + &origin_receipt, + request.holder_site_kind, + ) { + Ok(router_mesh_addr.to_string()) + } else { + origin_state + .router_mesh_addr + .as_deref() + .ok_or_else(|| { + ProtocolApiError::from(protocol_error( + ProtocolErrorCode::OriginUnavailable, + &format!( + "site `{}` does not expose a live router mesh address for dynamic \ + capability publication", + app.controller_plan.site_id + ), + )) + }) + .and_then(|router_mesh_addr| { + app.runtime + .router_mesh_addr_for_component_consumer( + app.controller_plan.kind, + request.holder_site_kind, + router_mesh_addr, + ) + .map_err(|err| { + ProtocolApiError::from(protocol_error( + ProtocolErrorCode::OriginUnavailable, + &format!( + "site `{}` exposes an invalid live router mesh address for \ + dynamic capability publication: {err}", + app.controller_plan.site_id + ), + )) + }) + }) + }?; let overlay_id = dynamic_caps::origin_overlay_id( &request.holder_component_id, &resolved_source.root_authority_selector, @@ -307,18 +316,18 @@ async fn resolve_dynamic_capability_origin( app: &ControlStateApp, request: dynamic_caps::ControlDynamicResolveOriginRequest, ) -> std::result::Result { - let holder_runtime = { - let state = app.control_state.lock().await.clone(); - local_component_runtime(app, &state, &request.holder_component_id)? - }; + let state = app.control_state.lock().await.clone(); + let holder_peer = local_component_runtime(app, &state, &request.holder_component_id)? + .mesh_config + .identity; resolve_dynamic_capability_origin_internal( app, dynamic_caps::InternalDynamicResolveOriginRequest { holder_component_id: request.holder_component_id, source: request.source, - holder_peer_id: holder_runtime.mesh_config.identity.id.clone(), + holder_peer_id: holder_peer.id, holder_peer_key_b64: base64::engine::general_purpose::STANDARD - .encode(holder_runtime.mesh_config.identity.public_key), + .encode(holder_peer.public_key), holder_site_kind: app.controller_plan.kind, }, ) diff --git a/runtime/site-controller/src/default_runtime.rs b/runtime/site-controller/src/default_runtime.rs index e204911b..2811961e 100644 --- a/runtime/site-controller/src/default_runtime.rs +++ b/runtime/site-controller/src/default_runtime.rs @@ -36,7 +36,6 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map as JsonMap, Value as JsonValue}; use sha2::{Digest as _, Sha256}; use tokio::{ - io::{AsyncReadExt as _, AsyncWriteExt as _}, sync::Mutex as AsyncMutex, time::{Instant, sleep}, }; @@ -67,11 +66,11 @@ pub use self::{ }; use self::{ child_runtime::{ - BridgeProxyHandle, BridgeProxyKey, DynamicComposeChildMetadata, SiteControllerRuntimeApp, - SiteControllerRuntimeChildRecord, SiteControllerRuntimeState, StoredRouteOverlayPayload, - dynamic_compose_child_metadata_path, dynamic_route_overlay_path, - load_dynamic_proxy_exports_metadata, site_controller_runtime_child_runtime_root, - site_controller_runtime_state_path, write_dynamic_route_overlay_payload, + DynamicComposeChildMetadata, SiteControllerRuntimeApp, SiteControllerRuntimeChildRecord, + SiteControllerRuntimeState, StoredRouteOverlayPayload, dynamic_compose_child_metadata_path, + dynamic_route_overlay_path, load_dynamic_proxy_exports_metadata, + site_controller_runtime_child_runtime_root, site_controller_runtime_state_path, + write_dynamic_route_overlay_payload, }, dynamic_routes::*, site_runtime_support::*, @@ -110,7 +109,6 @@ const DESIRED_LINKS_VERSION: u32 = 1; const DEFAULT_EXTERNAL_ENV_FILE: &str = "router-external.env"; const DEFAULT_K8S_OTEL_UPSTREAM: &str = "http://host.docker.internal:18890"; const CONTAINER_HOST_ALIAS: &str = "host.docker.internal"; -const PROCESS_SHUTDOWN_GRACE_PERIOD: Duration = Duration::from_secs(10); const KUBERNETES_WORKLOAD_READY_TIMEOUT: Duration = Duration::from_secs(120); const KUBERNETES_SITE_READY_BUFFER: Duration = Duration::from_secs(30); const COMPOSE_EGRESS_SUBNET_COUNT: u32 = 1 << 18; @@ -152,6 +150,10 @@ struct SiteManagerState { #[serde(default, skip_serializing_if = "Option::is_none")] router_mesh_addr: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + compose_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + kubernetes_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] router_identity_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] router_public_key_b64: Option, @@ -1115,10 +1117,10 @@ async fn wait_for_kubernetes_site_router_ready( ) -> Result<()> { debug_assert_eq!(plan.kind, SiteKind::Kubernetes); - let state_path = Path::new(&plan.site_state_root).join("manager-state.json"); let deadline = Instant::now() + timeout; loop { - if state_path.is_file() { + let state_path = Path::new(&plan.site_state_root).join("manager-state.json"); + let manager_state = if state_path.is_file() { let state: SiteManagerState = read_json(&state_path, "site manager state")?; if matches!(state.status, SiteLifecycleStatus::Failed) { return Err(miette::miette!( @@ -1129,35 +1131,18 @@ async fn wait_for_kubernetes_site_router_ready( .unwrap_or_else(|| "unknown failure".to_string()) )); } - if matches!(state.status, SiteLifecycleStatus::Running) - && let (Some(control), Some(mesh_addr)) = ( - state.router_control.as_deref(), - state.router_mesh_addr.as_deref(), - ) - { - let control_addr: SocketAddr = - control.parse().into_diagnostic().wrap_err_with(|| { - format!("invalid kubernetes router control addr `{control}`") - })?; - let mesh_addr: SocketAddr = - mesh_addr.parse().into_diagnostic().wrap_err_with(|| { - format!("invalid kubernetes router mesh addr `{mesh_addr}`") - })?; - if probe_kubernetes_router_identity(control_addr, Duration::from_millis(250)) - .await? - && router_mesh_listener_ready(mesh_addr).await - { - return Ok(()); - } - } - } else if let Some(control_target) = plan.local_router_control.as_deref() { - let mesh_target = kubernetes_local_router_mesh_target(plan)?; - if probe_kubernetes_router_identity_target(control_target, Duration::from_millis(250)) + Some(state) + } else { + None + }; + + if let Some((control_target, mesh_target)) = + kubernetes_router_ready_targets(plan, manager_state.as_ref())? + && probe_kubernetes_router_control_ready(&control_target, Duration::from_millis(250)) .await? - && router_mesh_listener_ready_target(&mesh_target).await - { - return Ok(()); - } + && router_mesh_listener_ready_target(&mesh_target).await + { + return Ok(()); } if Instant::now() >= deadline { return Err(miette::miette!( @@ -1169,6 +1154,40 @@ async fn wait_for_kubernetes_site_router_ready( } } +fn kubernetes_router_ready_targets( + plan: &SiteControllerRuntimePlan, + manager_state: Option<&SiteManagerState>, +) -> Result> { + if let Some(control_target) = plan.local_router_control.as_deref() { + return Ok(Some(( + control_target.to_string(), + kubernetes_local_router_mesh_target(plan)?, + ))); + } + + let Some(state) = manager_state else { + return Ok(None); + }; + if !matches!(state.status, SiteLifecycleStatus::Running) { + return Ok(None); + } + let (Some(control), Some(mesh_addr)) = ( + state.router_control.as_deref(), + state.router_mesh_addr.as_deref(), + ) else { + return Ok(None); + }; + let control_addr: SocketAddr = control + .parse() + .into_diagnostic() + .wrap_err_with(|| format!("invalid kubernetes router control addr `{control}`"))?; + let mesh_addr: SocketAddr = mesh_addr + .parse() + .into_diagnostic() + .wrap_err_with(|| format!("invalid kubernetes router mesh addr `{mesh_addr}`"))?; + Ok(Some((control_addr.to_string(), mesh_addr.to_string()))) +} + fn kubernetes_local_router_mesh_target(plan: &SiteControllerRuntimePlan) -> Result { let control_target = plan.local_router_control.as_deref().ok_or_else(|| { miette::miette!( @@ -1195,128 +1214,26 @@ async fn router_mesh_listener_ready_target(target: &str) -> bool { tokio::net::TcpStream::connect(target).await.is_ok() } -async fn probe_kubernetes_router_identity(addr: SocketAddr, timeout: Duration) -> Result { - probe_kubernetes_router_identity_target(&addr.to_string(), timeout).await -} - -async fn probe_kubernetes_router_identity_target(target: &str, timeout: Duration) -> Result { - let mut stream = - match tokio::time::timeout(timeout, tokio::net::TcpStream::connect(target)).await { - Ok(Ok(stream)) => stream, - Ok(Err(err)) - if matches!( - err.kind(), - std::io::ErrorKind::ConnectionRefused - | std::io::ErrorKind::ConnectionReset - | std::io::ErrorKind::ConnectionAborted - | std::io::ErrorKind::TimedOut - | std::io::ErrorKind::UnexpectedEof - | std::io::ErrorKind::NotConnected - ) => - { - return Ok(false); - } - Ok(Err(err)) => { - return Err(miette::miette!( - "failed to connect to kubernetes router control at {target}: {err}" - )); - } - Err(_) => return Ok(false), - }; - - let request = b"GET /identity HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"; - match tokio::time::timeout(timeout, stream.write_all(request)).await { - Ok(Ok(())) => {} - Ok(Err(err)) - if matches!( - err.kind(), - std::io::ErrorKind::BrokenPipe - | std::io::ErrorKind::ConnectionReset - | std::io::ErrorKind::ConnectionAborted - | std::io::ErrorKind::TimedOut - | std::io::ErrorKind::UnexpectedEof - | std::io::ErrorKind::NotConnected - ) => - { - return Ok(false); - } - Ok(Err(err)) => { +async fn probe_kubernetes_router_control_ready(target: &str, timeout: Duration) -> Result { + let url = format!("http://{target}/identity"); + let client = reqwest::Client::builder() + .timeout(timeout) + .build() + .into_diagnostic() + .wrap_err("failed to build kubernetes router readiness client")?; + let response = match client.get(&url).send().await { + Ok(response) => response, + Err(err) if err.is_connect() || err.is_timeout() => return Ok(false), + Err(err) => { return Err(miette::miette!( - "failed to write kubernetes router identity probe to {target}: {err}" + "failed to probe kubernetes router control at {target}: {err}" )); } - Err(_) => return Ok(false), - } - - let deadline = Instant::now() + timeout; - let mut buf = Vec::new(); - loop { - if let Some(end) = find_header_end(&buf) - && let Some(content_length) = router_identity_probe_content_length(&buf[..end])? - { - let body_len = buf.len().saturating_sub(end + 4); - if body_len >= content_length { - return parse_router_identity_probe_response(&buf, end); - } - } - let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - return Ok(false); - }; - let mut chunk = [0u8; 1024]; - match tokio::time::timeout(remaining, stream.read(&mut chunk)).await { - Ok(Ok(0)) => return Ok(false), - Ok(Ok(read)) => buf.extend_from_slice(&chunk[..read]), - Ok(Err(err)) - if matches!( - err.kind(), - std::io::ErrorKind::ConnectionReset - | std::io::ErrorKind::ConnectionAborted - | std::io::ErrorKind::TimedOut - | std::io::ErrorKind::UnexpectedEof - | std::io::ErrorKind::NotConnected - ) => - { - return Ok(false); - } - Ok(Err(err)) => { - return Err(miette::miette!( - "failed to read kubernetes router identity probe from {target}: {err}" - )); - } - Err(_) => return Ok(false), - } - } -} - -fn router_identity_probe_content_length(header: &[u8]) -> Result> { - let header = std::str::from_utf8(header) - .into_diagnostic() - .wrap_err("router identity probe returned a non-UTF-8 HTTP header")?; - Ok(header.lines().find_map(|line| { - let (name, value) = line.split_once(':')?; - name.trim() - .eq_ignore_ascii_case("content-length") - .then_some(value.trim()) - .and_then(|value| value.parse::().ok()) - })) -} - -fn parse_router_identity_probe_response(buf: &[u8], header_end: usize) -> Result { - let header = std::str::from_utf8(&buf[..header_end]) - .into_diagnostic() - .wrap_err("router identity probe returned a non-UTF-8 HTTP header")?; - let status = header - .lines() - .next() - .and_then(|line| line.split_whitespace().nth(1)) - .and_then(|code| code.parse::().ok()); - if status != Some(200) { + }; + if !response.status().is_success() { return Ok(false); } - let body = std::str::from_utf8(&buf[header_end + 4..]) - .into_diagnostic() - .wrap_err("router identity probe returned a non-UTF-8 body")?; - Ok(serde_json::from_str::(body.trim()).is_ok()) + Ok(response.json::().await.is_ok()) } fn filter_direct_stage_plan(artifact_root: &Path, component_ids: &[usize]) -> Result<()> { @@ -1902,3 +1819,127 @@ fn terminate_detached_runtime(root_pid: u32, timeout: Duration) -> Result<()> { .join(", ") )) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn kubernetes_router_ready_prefers_local_targets_over_manager_state() { + let plan = SiteControllerRuntimePlan { + schema: "amber.run.site_controller_runtime_plan".to_string(), + version: 1, + run_id: "run".to_string(), + mesh_scope: "scope".to_string(), + run_root: "/tmp/run".to_string(), + site_id: "kind_local".to_string(), + kind: SiteKind::Kubernetes, + router_identity_id: "/site/test/router".to_string(), + local_router_control: Some("amber-router:24100".to_string()), + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), + listen_addr: SocketAddr::from(([127, 0, 0, 1], 4100)), + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: Some("ns".to_string()), + context: None, + observability_endpoint: None, + launch_env: BTreeMap::new(), + }; + let stale_manager_state = SiteManagerState { + schema: SITE_PLAN_SCHEMA.to_string(), + version: SITE_PLAN_VERSION, + run_id: "run".to_string(), + site_id: "kind_local".to_string(), + kind: SiteKind::Kubernetes, + status: SiteLifecycleStatus::Running, + artifact_dir: "/tmp/artifact".to_string(), + supervisor_pid: 1, + process_pid: None, + compose_project: None, + kubernetes_namespace: Some("ns".to_string()), + port_forward_pid: None, + context: None, + router_control: Some("127.0.0.1:9".to_string()), + router_mesh_addr: Some("127.0.0.1:9".to_string()), + compose_consumer_router_mesh_addr: None, + kubernetes_consumer_router_mesh_addr: None, + router_identity_id: None, + router_public_key_b64: None, + site_controller_pid: None, + site_controller_url: None, + last_error: None, + }; + + assert_eq!( + kubernetes_router_ready_targets(&plan, Some(&stale_manager_state)) + .expect("targets should resolve"), + Some(( + "amber-router:24100".to_string(), + "amber-router:24000".to_string() + )), + "embedded kubernetes controllers must prefer their local router service over stale \ + manager-state loopback endpoints", + ); + } + + #[test] + fn kubernetes_router_ready_falls_back_to_manager_state_when_local_target_is_absent() { + let plan = SiteControllerRuntimePlan { + schema: "amber.run.site_controller_runtime_plan".to_string(), + version: 1, + run_id: "run".to_string(), + mesh_scope: "scope".to_string(), + run_root: "/tmp/run".to_string(), + site_id: "kind_local".to_string(), + kind: SiteKind::Kubernetes, + router_identity_id: "/site/test/router".to_string(), + local_router_control: None, + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), + listen_addr: SocketAddr::from(([127, 0, 0, 1], 4100)), + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: Some("ns".to_string()), + context: None, + observability_endpoint: None, + launch_env: BTreeMap::new(), + }; + let manager_state = SiteManagerState { + schema: SITE_PLAN_SCHEMA.to_string(), + version: SITE_PLAN_VERSION, + run_id: "run".to_string(), + site_id: "kind_local".to_string(), + kind: SiteKind::Kubernetes, + status: SiteLifecycleStatus::Running, + artifact_dir: "/tmp/artifact".to_string(), + supervisor_pid: 1, + process_pid: None, + compose_project: None, + kubernetes_namespace: Some("ns".to_string()), + port_forward_pid: None, + context: None, + router_control: Some("127.0.0.1:24100".to_string()), + router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: None, + kubernetes_consumer_router_mesh_addr: None, + router_identity_id: None, + router_public_key_b64: None, + site_controller_pid: None, + site_controller_url: None, + last_error: None, + }; + + assert_eq!( + kubernetes_router_ready_targets(&plan, Some(&manager_state)) + .expect("targets should resolve"), + Some(("127.0.0.1:24100".to_string(), "127.0.0.1:24000".to_string())), + "host-supervised kubernetes sites should still fall back to manager-state endpoints \ + when no embedded local router target is available", + ); + } +} diff --git a/runtime/site-controller/src/default_runtime/child_runtime.rs b/runtime/site-controller/src/default_runtime/child_runtime.rs index b12c0a5b..3adac966 100644 --- a/runtime/site-controller/src/default_runtime/child_runtime.rs +++ b/runtime/site-controller/src/default_runtime/child_runtime.rs @@ -44,21 +44,8 @@ pub(super) struct StoredRouteOverlayPayload { pub(super) inbound_routes: Vec, } -pub(crate) struct BridgeProxyHandle { - pub(crate) child: Child, - pub(crate) listen: SocketAddr, -} - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub(crate) struct BridgeProxyKey { - pub(crate) provider_output_dir: String, - pub(crate) export_name: String, - pub(crate) consumer_kind: SiteKind, -} - #[derive(Clone, Default)] pub struct DefaultSiteControllerRuntime { - bridge_proxies: Arc>>, runtime_apps: Arc>>, } @@ -69,15 +56,11 @@ pub(crate) fn default_site_controller_runtime() -> SharedSiteControllerRuntime { impl SiteControllerRuntime for DefaultSiteControllerRuntime { fn cleanup<'a>(&'a self) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async move { - let mut bridge_proxies = { - let mut guard = self.bridge_proxies.lock().await; - std::mem::take(&mut *guard) - }; self.runtime_apps .lock() .expect("site controller runtime app cache poisoned") .clear(); - stop_bridge_proxies(&mut bridge_proxies).await + Ok(()) }) } @@ -90,14 +73,12 @@ impl SiteControllerRuntime for DefaultSiteControllerRuntime { run_root: &'a Path, ) -> SiteControllerRuntimeFuture<'a, String> { Box::pin(async move { - let mut bridge_proxies = self.bridge_proxies.lock().await; resolve_link_external_url_for_output( provider, provider_output_dir, link, consumer_kind, run_root, - &mut bridge_proxies, ) .await }) @@ -174,6 +155,15 @@ impl SiteControllerRuntime for DefaultSiteControllerRuntime { router_mesh_addr_for_consumer(provider_kind, consumer_kind, router_mesh_addr) } + fn router_mesh_addr_for_component_consumer( + &self, + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> Result { + router_mesh_addr_for_component_consumer(provider_kind, consumer_kind, router_mesh_addr) + } + fn update_desired_overlay_for_consumer( &self, site_state_root: &Path, @@ -234,16 +224,6 @@ impl DefaultSiteControllerRuntime { } } -pub(crate) async fn stop_bridge_proxies( - bridge_proxies: &mut BTreeMap, -) -> Result<()> { - for bridge in bridge_proxies.values_mut() { - stop_child(&mut bridge.child).await?; - } - bridge_proxies.clear(); - Ok(()) -} - pub(super) fn build_site_controller_runtime_app( plan: SiteControllerRuntimePlan, ) -> Result { @@ -411,12 +391,11 @@ pub fn cleanup_dynamic_site_children(site_state_root: &Path, kind: SiteKind) -> if let Some(pid) = child.process_pid { terminate_pid(pid, site_ready_timeout_for_kind(kind))?; } - remove_dir_if_exists( - &crate::runtime_api::site_controller_runtime_child_root_for_site( - site_state_root, - child.child_id, - ), - )?; + let child_root = crate::runtime_api::site_controller_runtime_child_root_for_site( + site_state_root, + child.child_id, + ); + remove_dynamic_child_root(kind, &child_root, Some(Path::new(&child.artifact_root)))?; } if state.children.is_empty() { return Ok(()); @@ -460,6 +439,130 @@ fn cleanup_dynamic_compose_child( Ok(()) } +fn compose_cleanup_image_candidates(artifact_root: &Path) -> Result> { + let compose_path = artifact_root.join("compose.yaml"); + if !compose_path.is_file() { + return Ok(vec![ + "docker:28-cli".to_string(), + "python:3.13-alpine".to_string(), + ]); + } + + let document = read_compose_document(&compose_path)?; + let Some(root) = document.as_mapping() else { + return Err(miette::miette!( + "compose file {} is not a YAML mapping", + compose_path.display() + )); + }; + let Some(services) = root + .get(yaml_string("services")) + .and_then(serde_yaml::Value::as_mapping) + else { + return Err(miette::miette!( + "compose file {} is missing a services mapping", + compose_path.display() + )); + }; + + let mut images = BTreeSet::from([ + "docker:28-cli".to_string(), + "python:3.13-alpine".to_string(), + ]); + for service in services.values() { + let Some(service_mapping) = service.as_mapping() else { + continue; + }; + let Some(image) = service_mapping + .get(yaml_string("image")) + .and_then(serde_yaml::Value::as_str) + else { + continue; + }; + images.insert(image.to_string()); + } + Ok(images.into_iter().collect()) +} + +fn docker_image_exists_locally(image: &str) -> bool { + Command::new("docker") + .arg("image") + .arg("inspect") + .arg(image) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()) +} + +fn force_remove_dir_if_exists_via_local_container( + path: &Path, + artifact_root: &Path, +) -> Result { + if !path.exists() { + return Ok(true); + } + let Some(parent) = path.parent() else { + return Ok(false); + }; + + for image in compose_cleanup_image_candidates(artifact_root)? { + if !docker_image_exists_locally(&image) { + continue; + } + let status = Command::new("docker") + .arg("run") + .arg("--rm") + .arg("--user") + .arg("0:0") + .arg("-v") + .arg(format!("{}:{}", parent.display(), parent.display())) + .arg("--entrypoint") + .arg("rm") + .arg(&image) + .arg("-rf") + .arg(path) + .status() + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to launch fallback cleanup container `{image}` for {}", + path.display() + ) + })?; + if status.success() { + return Ok(true); + } + } + Ok(false) +} + +fn remove_dynamic_child_root( + kind: SiteKind, + child_root: &Path, + artifact_root: Option<&Path>, +) -> Result<()> { + match remove_dir_if_exists(child_root) { + Ok(()) => Ok(()), + Err(err) if matches!(kind, SiteKind::Compose) => { + let Some(artifact_root) = artifact_root else { + return Err(err.wrap_err(format!( + "compose child root {} is missing its artifact directory", + child_root.display() + ))); + }; + if force_remove_dir_if_exists_via_local_container(child_root, artifact_root)? { + return Ok(()); + } + Err(err.wrap_err(format!( + "compose child root {} could not be removed with a local cleanup container", + child_root.display() + ))) + } + Err(err) => Err(err), + } +} + pub(super) async fn site_controller_runtime_prepare_child( app: &SiteControllerRuntimeApp, state: &FrameworkControlState, @@ -892,6 +995,20 @@ pub(super) async fn site_controller_runtime_publish_child( ) .await?; } + { + let mut state = app.state.lock().await; + let record = state + .children + .get_mut(&child_id) + .expect("prepared child should remain present"); + // Compose workloads can make one-shot startup calls through their sidecars. + // Mark the child publishable as soon as its sidecars are up so same-site + // direct-input grants are reconciled before the workload process starts. + record.published = true; + write_json(&app.state_path, &*state)?; + } + reconcile_dynamic_site_router_overlays(app).await?; + reconcile_dynamic_direct_input_overlays(app).await?; if !workload_services.is_empty() { let status = compose_command(Some(compose_project), Path::new(&child.artifact_root)) @@ -921,16 +1038,6 @@ pub(super) async fn site_controller_runtime_publish_child( site_ready_timeout_for_kind(SiteKind::Compose), ) .await?; - let mut state = app.state.lock().await; - let record = state - .children - .get_mut(&child_id) - .expect("prepared child should remain present"); - record.published = true; - write_json(&app.state_path, &*state)?; - drop(state); - reconcile_dynamic_site_router_overlays(app).await?; - reconcile_dynamic_direct_input_overlays(app).await?; } SiteKind::Kubernetes => { let artifact_root = Path::new(&child.artifact_root); @@ -1001,7 +1108,12 @@ pub(super) async fn site_controller_runtime_rollback_child( if let Some(pid) = child.process_pid { terminate_pid(pid, site_ready_timeout_for_kind(app.plan.kind))?; } - remove_dir_if_exists(&site_controller_runtime_child_root(&app.plan, child_id)) + let child_root = site_controller_runtime_child_root(&app.plan, child_id); + remove_dynamic_child_root( + app.plan.kind, + &child_root, + Some(Path::new(&child.artifact_root)), + ) } pub(super) async fn site_controller_runtime_destroy_child( @@ -1015,7 +1127,9 @@ pub(super) async fn site_controller_runtime_destroy_child( state.children.get(&child_id).cloned() }; if child.is_none() { - remove_dir_if_exists(&site_controller_runtime_child_root(&app.plan, child_id))?; + let child_root = site_controller_runtime_child_root(&app.plan, child_id); + let artifact_root = child_root.join("artifact"); + remove_dynamic_child_root(app.plan.kind, &child_root, Some(&artifact_root))?; return Ok(()); } if let Some(child) = child.as_ref() @@ -1130,5 +1244,38 @@ pub(super) async fn site_controller_runtime_destroy_child( reconcile_dynamic_site_router_overlays(app).await?; reconcile_dynamic_direct_input_overlays(app).await?; } - remove_dir_if_exists(&site_controller_runtime_child_root(&app.plan, child_id)) + let child_root = site_controller_runtime_child_root(&app.plan, child_id); + remove_dynamic_child_root( + app.plan.kind, + &child_root, + child.as_ref().map(|child| Path::new(&child.artifact_root)), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn compose_cleanup_image_candidates_include_service_images_and_fallbacks() { + let temp = tempfile::tempdir().expect("temp dir"); + fs::write( + temp.path().join("compose.yaml"), + "services:\n app:\n image: python:3.13-alpine\n helper:\n image: \ + ghcr.io/rdi-foundation/amber-helper:test-tag\n", + ) + .expect("compose file should write"); + + let images = compose_cleanup_image_candidates(temp.path()) + .expect("compose cleanup images should load"); + + assert_eq!( + images, + vec![ + "docker:28-cli".to_string(), + "ghcr.io/rdi-foundation/amber-helper:test-tag".to_string(), + "python:3.13-alpine".to_string(), + ], + ); + } } diff --git a/runtime/site-controller/src/default_runtime/compose_controller.rs b/runtime/site-controller/src/default_runtime/compose_controller.rs index 5b02d9af..1a8b11b6 100644 --- a/runtime/site-controller/src/default_runtime/compose_controller.rs +++ b/runtime/site-controller/src/default_runtime/compose_controller.rs @@ -1,3 +1,5 @@ +#[cfg(all(unix, not(target_os = "macos")))] +use std::os::unix::fs::MetadataExt; use std::{fs, path::Path}; use serde_json::json; @@ -12,6 +14,58 @@ const COMPOSE_ROUTER_CONTROL_SOCKET_DIR: &str = "/amber/control"; const COMPOSE_ROUTER_CONTROL_VOLUME_NAME: &str = "amber-router-control"; const DOCKER_SOCK_PATH: &str = "/var/run/docker.sock"; +#[cfg(target_os = "macos")] +fn compose_site_controller_user() -> String { + // Docker Desktop's socket mount on macOS is mediated by the host, so the controller needs + // root inside the Compose site to reach it reliably. + "0:0".to_string() +} + +#[cfg(all(unix, not(target_os = "macos")))] +fn compose_site_controller_user() -> String { + if fs::metadata(DOCKER_SOCK_PATH).is_ok() { + return format!("{}:{}", unsafe { libc::geteuid() }, unsafe { + libc::getegid() + }); + } + "0:0".to_string() +} + +#[cfg(not(unix))] +fn compose_site_controller_user() -> String { + "0:0".to_string() +} + +#[cfg(target_os = "macos")] +fn compose_site_controller_group_add() -> Vec { + Vec::new() +} + +fn compose_site_controller_env(plan: &SiteControllerPlan) -> serde_json::Value { + serde_json::Value::Object( + plan.launch_env + .iter() + .map(|(key, value)| (key.clone(), serde_json::Value::String(value.clone()))) + .collect(), + ) +} + +#[cfg(all(unix, not(target_os = "macos")))] +fn compose_site_controller_group_add() -> Vec { + if let Ok(metadata) = fs::metadata(DOCKER_SOCK_PATH) { + let socket_gid = metadata.gid(); + if socket_gid != unsafe { libc::getegid() } { + return vec![socket_gid.to_string()]; + } + } + Vec::new() +} + +#[cfg(not(unix))] +fn compose_site_controller_group_add() -> Vec { + Vec::new() +} + pub fn inject_compose_site_controller( artifact_root: &Path, plan: &SiteControllerPlan, @@ -49,10 +103,13 @@ pub fn inject_compose_site_controller( COMPOSE_MESH_NETWORK_NAME.to_string(), serde_json::Value::Object(serde_json::Map::new()), )]); + let group_add = compose_site_controller_group_add(); let service = json!({ "image": controller_image, - "user": "0:0", + "user": compose_site_controller_user(), + "group_add": group_add, "command": ["--plan", plan_path.display().to_string()], + "environment": compose_site_controller_env(plan), "networks": networks, "extra_hosts": ["host.docker.internal:host-gateway"], "healthcheck": { @@ -89,3 +146,93 @@ pub fn inject_compose_site_controller( .into_diagnostic() .wrap_err_with(|| format!("failed to write {}", compose_path.display())) } + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, net::SocketAddr}; + + use amber_compiler::run_plan::SiteKind; + + use super::*; + + fn test_plan(run_root: &Path) -> SiteControllerPlan { + SiteControllerPlan { + schema: "amber.framework_component.site_controller_plan".to_string(), + version: 1, + run_id: "run-test".to_string(), + mesh_scope: "scope".to_string(), + site_id: "compose_local".to_string(), + kind: SiteKind::Compose, + listen_addr: SocketAddr::from(([127, 0, 0, 1], 4100)), + authority_url: "http://amber-site-controller:4100".to_string(), + router_identity_id: "/site/compose_local/router".to_string(), + peer_site_router_urls: BTreeMap::new(), + peer_router_identities: BTreeMap::new(), + peer_router_mesh_addrs: BTreeMap::new(), + local_router_control: Some("unix:///tmp/router.sock".to_string()), + published_router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: Some("host.docker.internal:24000".to_string()), + kubernetes_consumer_router_mesh_addr: Some("192.168.65.254:24000".to_string()), + state_path: run_root.join("state.json").display().to_string(), + run_root: run_root.display().to_string(), + state_root: run_root.join("state-root").display().to_string(), + site_state_root: run_root.join("site-state").display().to_string(), + artifact_dir: run_root.join("artifact").display().to_string(), + auth_token: "token".to_string(), + dynamic_caps_token_verify_key_b64: "verify".to_string(), + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24000), + compose_project: Some("compose-project".to_string()), + kubernetes_namespace: None, + context: None, + observability_endpoint: None, + launch_env: BTreeMap::from([( + "AMBER_DEV_IMAGE_TAGS".to_string(), + "router=dev-tag,helper=dev-tag".to_string(), + )]), + } + } + + #[test] + fn inject_compose_site_controller_propagates_launch_env() { + let temp = tempfile::tempdir().expect("temp dir"); + let compose_path = temp.path().join("compose.yaml"); + fs::write( + &compose_path, + "services:\n amber-router:\n image: ghcr.io/rdi-foundation/amber-router:test\n", + ) + .expect("compose file should write"); + + let plan = test_plan(temp.path()); + let plan_path = temp.path().join("site-controller-plan.json"); + fs::write(&plan_path, "{}").expect("plan file should write"); + + inject_compose_site_controller( + temp.path(), + &plan, + &plan_path, + "ghcr.io/rdi-foundation/amber-site-controller:test", + ) + .expect("compose site controller injection should succeed"); + + let document = read_compose_document(&compose_path).expect("compose should parse"); + let service = document + .as_mapping() + .and_then(|root| root.get(yaml_string("services"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|services| services.get(yaml_string(SITE_CONTROLLER_SERVICE_NAME))) + .and_then(serde_yaml::Value::as_mapping) + .expect("site controller service should exist"); + let environment = service + .get(yaml_string("environment")) + .and_then(serde_yaml::Value::as_mapping) + .expect("site controller service should include environment"); + assert_eq!( + environment + .get(yaml_string("AMBER_DEV_IMAGE_TAGS")) + .and_then(serde_yaml::Value::as_str), + Some("router=dev-tag,helper=dev-tag"), + ); + } +} diff --git a/runtime/site-controller/src/default_runtime/dynamic_routes.rs b/runtime/site-controller/src/default_runtime/dynamic_routes.rs index 6e381926..1e949e19 100644 --- a/runtime/site-controller/src/default_runtime/dynamic_routes.rs +++ b/runtime/site-controller/src/default_runtime/dynamic_routes.rs @@ -459,7 +459,7 @@ pub(super) fn rewrite_dynamic_direct_inputs( matched = true; route.route_id = route_id.clone(); route.protocol = protocol; - route.peer_addr = provider_runtime.host_mesh_addr.clone(); + route.peer_addr = provider_runtime.component_reachable_mesh_addr.clone(); route.peer_id = provider_runtime.mesh_config.identity.id.clone(); route.capability = capability.clone(); route.capability_kind = Some(input.capability_kind.clone()); @@ -1448,7 +1448,8 @@ mod direct_input_tests { "/provider".to_string(), LiveComponentRuntimeMetadata { moniker: "/provider".to_string(), - host_mesh_addr: "10.0.0.20:24001".to_string(), + router_reachable_mesh_addr: "127.0.0.1:24001".to_string(), + component_reachable_mesh_addr: "10.0.0.20:24001".to_string(), control_endpoint: None, mesh_config: MeshConfigPublic { identity: MeshIdentityPublic { diff --git a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs index 7cac53cd..e018807f 100644 --- a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs +++ b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs @@ -35,6 +35,13 @@ const KUBERNETES_CONTROLLER_STATE_PATH: &str = "/amber/site/state/site-controlle const KUBERNETES_CONTROLLER_DESIRED_LINKS_PATH: &str = "/amber/site/state/desired-links.json"; const KUBERNETES_CONTROLLER_SEED_ROOT: &str = "/amber/seed"; +fn kubernetes_env_entries(plan: &SiteControllerPlan) -> Vec { + plan.launch_env + .iter() + .map(|(name, value)| json!({ "name": name, "value": value })) + .collect() +} + pub fn inject_kubernetes_site_controller( artifact_root: &Path, plan: &SiteControllerPlan, @@ -45,6 +52,7 @@ pub fn inject_kubernetes_site_controller( build_kubernetes_controller_seed_configmap(artifact_root, plan, &embedded_plan)?; let labels = kubernetes_controller_labels(); let selector = kubernetes_controller_selector(); + let controller_env = kubernetes_env_entries(plan); write_yaml_artifact( artifact_root.join(KUBERNETES_CONTROLLER_SEED_CONFIGMAP_PATH), @@ -148,7 +156,7 @@ pub fn inject_kubernetes_site_controller( "automountServiceAccountToken": true, "initContainers": [{ "name": "seed-site-controller", - "image": "busybox:1.36.1", + "image": controller_image, "command": [ "sh", "-lc", @@ -181,6 +189,7 @@ pub fn inject_kubernetes_site_controller( "name": SITE_CONTROLLER_SERVICE_NAME, "image": controller_image, "args": ["--plan", KUBERNETES_CONTROLLER_PLAN_PATH], + "env": controller_env, "ports": [{ "name": "http", "containerPort": SITE_CONTROLLER_PORT, @@ -559,3 +568,121 @@ fn write_yaml_artifact(path: PathBuf, value: &serde_json::Value) -> Result<()> { .into_diagnostic() .wrap_err_with(|| format!("failed to write {}", path.display())) } + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, net::SocketAddr}; + + use amber_compiler::run_plan::SiteKind; + + use super::*; + + fn test_plan(root: &Path) -> SiteControllerPlan { + SiteControllerPlan { + schema: "amber.framework_component.site_controller_plan".to_string(), + version: 1, + run_id: "run-test".to_string(), + mesh_scope: "scope".to_string(), + site_id: "kind_local".to_string(), + kind: SiteKind::Kubernetes, + listen_addr: SocketAddr::from(([127, 0, 0, 1], 4100)), + authority_url: "http://amber-site-controller:4100".to_string(), + router_identity_id: "/site/kind_local/router".to_string(), + peer_site_router_urls: BTreeMap::new(), + peer_router_identities: BTreeMap::new(), + peer_router_mesh_addrs: BTreeMap::new(), + local_router_control: Some("amber-router:24100".to_string()), + published_router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: Some("host.docker.internal:24000".to_string()), + kubernetes_consumer_router_mesh_addr: Some("192.168.65.254:24000".to_string()), + state_path: root.join("state.json").display().to_string(), + run_root: root.join("run").display().to_string(), + state_root: root.join("state-root").display().to_string(), + site_state_root: root.join("site-state").display().to_string(), + artifact_dir: root.join("artifact").display().to_string(), + auth_token: "token".to_string(), + dynamic_caps_token_verify_key_b64: "verify".to_string(), + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: Some("test-ns".to_string()), + context: Some("test-context".to_string()), + observability_endpoint: None, + launch_env: BTreeMap::from([( + "AMBER_DEV_IMAGE_TAGS".to_string(), + "router=dev-tag,helper=dev-tag".to_string(), + )]), + } + } + + #[test] + fn inject_kubernetes_site_controller_propagates_launch_env() { + let temp = tempfile::tempdir().expect("temp dir"); + let artifact_root = temp.path(); + fs::create_dir_all(artifact_root.join("04-services")).expect("services dir"); + fs::create_dir_all(artifact_root.join("05-networkpolicies")).expect("netpol dir"); + fs::write( + artifact_root.join("kustomization.yaml"), + "resources:\n - 04-services/amber-router.yaml\n - \ + 05-networkpolicies/amber-router-netpol.yaml\n", + ) + .expect("kustomization should write"); + fs::write( + artifact_root.join(KUBERNETES_ROUTER_SERVICE_PATH), + "apiVersion: v1\nkind: Service\nmetadata:\n name: amber-router\nspec:\n ports:\n \ + - name: mesh\n port: 24000\n targetPort: 24000\n protocol: TCP\n", + ) + .expect("router service should write"); + fs::write( + artifact_root.join(KUBERNETES_ROUTER_NETPOL_PATH), + "apiVersion: networking.k8s.io/v1\nkind: NetworkPolicy\nmetadata:\n name: \ + amber-router-netpol\nspec:\n ingress: []\n", + ) + .expect("router netpol should write"); + + let plan = test_plan(artifact_root); + fs::write(&plan.state_path, "{}").expect("state should write"); + fs::create_dir_all(&plan.site_state_root).expect("site state dir"); + fs::write( + super::desired_links_path(Path::new(&plan.site_state_root)), + "{}", + ) + .expect("desired links should write"); + + inject_kubernetes_site_controller( + artifact_root, + &plan, + "ghcr.io/rdi-foundation/amber-site-controller:test", + ) + .expect("kubernetes site controller injection should succeed"); + + let deployment_raw = + fs::read_to_string(artifact_root.join(KUBERNETES_CONTROLLER_DEPLOYMENT_PATH)) + .expect("controller deployment should exist"); + let deployment: serde_yaml::Value = + serde_yaml::from_str(&deployment_raw).expect("deployment yaml should parse"); + let env = deployment + .as_mapping() + .and_then(|root| root.get(yaml_string("spec"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|spec| spec.get(yaml_string("template"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|template| template.get(yaml_string("spec"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|spec| spec.get(yaml_string("containers"))) + .and_then(serde_yaml::Value::as_sequence) + .and_then(|containers| containers.first()) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|container| container.get(yaml_string("env"))) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller container should include env"); + assert!(env.iter().any(|entry| { + entry.as_mapping().is_some_and(|mapping| { + mapping.get(yaml_string("name")) == Some(&yaml_string("AMBER_DEV_IMAGE_TAGS")) + && mapping.get(yaml_string("value")) + == Some(&yaml_string("router=dev-tag,helper=dev-tag")) + }) + })); + } +} diff --git a/runtime/site-controller/src/default_runtime/site_artifacts.rs b/runtime/site-controller/src/default_runtime/site_artifacts.rs index adc3e836..cb5cf6ae 100644 --- a/runtime/site-controller/src/default_runtime/site_artifacts.rs +++ b/runtime/site-controller/src/default_runtime/site_artifacts.rs @@ -51,6 +51,21 @@ fn kubernetes_component_mesh_peer_addr( Ok(format!("{service_name}:{mesh_port}")) } +fn local_component_mesh_peer_addr(mesh_port: u16) -> String { + #[cfg(target_os = "linux")] + { + std::net::SocketAddr::from((std::net::Ipv4Addr::new(10, 0, 2, 2), mesh_port)).to_string() + } + #[cfg(not(target_os = "linux"))] + { + std::net::SocketAddr::from(([127, 0, 0, 1], mesh_port)).to_string() + } +} + +fn local_router_mesh_peer_addr(mesh_port: u16) -> String { + std::net::SocketAddr::from(([127, 0, 0, 1], mesh_port)).to_string() +} + fn build_dynamic_compose_route_overlay_payload( artifact_root: &Path, assigned_components: &[String], @@ -597,7 +612,8 @@ fn collect_direct_artifact_runtime_metadata( component.moniker.clone(), LiveComponentRuntimeMetadata { moniker: component.moniker.clone(), - host_mesh_addr: format!("127.0.0.1:{mesh_port}"), + router_reachable_mesh_addr: local_router_mesh_peer_addr(mesh_port), + component_reachable_mesh_addr: local_component_mesh_peer_addr(mesh_port), control_endpoint: Some(ControlEndpoint::Unix( direct_component_control_socket_path( runtime_root.join(&component.program.work_dir), @@ -637,7 +653,8 @@ fn collect_vm_artifact_runtime_metadata( component.moniker.clone(), LiveComponentRuntimeMetadata { moniker: component.moniker.clone(), - host_mesh_addr: format!("127.0.0.1:{mesh_port}"), + router_reachable_mesh_addr: local_router_mesh_peer_addr(mesh_port), + component_reachable_mesh_addr: local_component_mesh_peer_addr(mesh_port), control_endpoint: Some(ControlEndpoint::Unix(vm_component_control_socket_path( runtime_root .join("work") @@ -722,7 +739,13 @@ fn collect_compose_artifact_runtime_metadata( target.config.identity.id.clone(), LiveComponentRuntimeMetadata { moniker: target.config.identity.id.clone(), - host_mesh_addr: compose_component_mesh_peer_addr( + router_reachable_mesh_addr: compose_component_mesh_peer_addr( + artifact_root, + &target.config.identity.id, + &target.output, + mesh_config.mesh_listen.port(), + )?, + component_reachable_mesh_addr: compose_component_mesh_peer_addr( artifact_root, &target.config.identity.id, &target.output, @@ -750,6 +773,14 @@ mod tests { use super::*; + fn expected_local_component_mesh_peer_addr(mesh_port: u16) -> String { + local_component_mesh_peer_addr(mesh_port) + } + + fn expected_local_router_mesh_peer_addr(mesh_port: u16) -> String { + local_router_mesh_peer_addr(mesh_port) + } + #[test] fn direct_runtime_metadata_uses_bounded_sidecar_control_socket_paths() { let work_dir = PathBuf::from( @@ -871,6 +902,16 @@ mod tests { let Some(ControlEndpoint::Unix(path)) = provider.control_endpoint.as_ref() else { panic!("direct metadata should expose a unix control socket"); }; + assert_eq!( + provider.router_reachable_mesh_addr, + expected_local_router_mesh_peer_addr(24001), + "direct runtime metadata should expose the router-reachable mesh address", + ); + assert_eq!( + provider.component_reachable_mesh_addr, + expected_local_component_mesh_peer_addr(24001), + "direct runtime metadata should expose the sidecar-reachable mesh address", + ); assert_eq!( path, &amber_mesh::stable_temp_socket_path( @@ -953,6 +994,16 @@ mod tests { let Some(ControlEndpoint::Unix(path)) = provider.control_endpoint.as_ref() else { panic!("vm metadata should expose a unix control socket"); }; + assert_eq!( + provider.router_reachable_mesh_addr, + expected_local_router_mesh_peer_addr(24001), + "vm runtime metadata should expose the router-reachable mesh address", + ); + assert_eq!( + provider.component_reachable_mesh_addr, + expected_local_component_mesh_peer_addr(24001), + "vm runtime metadata should expose the sidecar-reachable mesh address", + ); assert_eq!( path, &amber_mesh::stable_temp_socket_path( @@ -1138,7 +1189,13 @@ fn collect_kubernetes_artifact_runtime_metadata( target.config.identity.id.clone(), LiveComponentRuntimeMetadata { moniker: target.config.identity.id.clone(), - host_mesh_addr: kubernetes_component_mesh_peer_addr( + router_reachable_mesh_addr: kubernetes_component_mesh_peer_addr( + artifact_root, + &target.config.identity.id, + &target.output, + mesh_config.mesh_listen.port(), + )?, + component_reachable_mesh_addr: kubernetes_component_mesh_peer_addr( artifact_root, &target.config.identity.id, &target.output, diff --git a/runtime/site-controller/src/default_runtime/site_runtime_support.rs b/runtime/site-controller/src/default_runtime/site_runtime_support.rs index 0288ece7..c616c400 100644 --- a/runtime/site-controller/src/default_runtime/site_runtime_support.rs +++ b/runtime/site-controller/src/default_runtime/site_runtime_support.rs @@ -774,66 +774,6 @@ pub(crate) fn spawn_detached_child( }) } -pub(super) async fn stop_child(child: &mut Child) -> Result<()> { - #[cfg(unix)] - { - terminate_recorded_processes(&[child.id()]).await?; - let _ = child.wait(); - Ok(()) - } - - #[cfg(not(unix))] - { - send_sigterm(child.id()); - let _ = wait_for_child_exit(child, PROCESS_SHUTDOWN_GRACE_PERIOD).await; - Ok(()) - } -} - -#[cfg(not(unix))] -pub(super) async fn wait_for_child_exit(child: &mut Child, timeout: Duration) -> Result<()> { - let deadline = Instant::now() + timeout; - loop { - if child.try_wait().into_diagnostic()?.is_some() { - return Ok(()); - } - if Instant::now() >= deadline { - let _ = child.kill(); - let _ = child.wait(); - return Ok(()); - } - sleep(Duration::from_millis(100)).await; - } -} - -pub(super) async fn wait_for_pid_exit(pid: u32, timeout: Duration) -> bool { - let deadline = Instant::now() + timeout; - loop { - if !pid_is_alive(pid) { - return true; - } - if Instant::now() >= deadline { - return false; - } - sleep(Duration::from_millis(100)).await; - } -} - -pub(super) async fn router_mesh_listener_ready(addr: SocketAddr) -> bool { - tokio::net::TcpStream::connect(addr).await.is_ok() -} - -pub(super) async fn wait_for_socket_listener(addr: SocketAddr) -> Result<()> { - let deadline = Instant::now() + Duration::from_secs(30); - while Instant::now() < deadline { - if router_mesh_listener_ready(addr).await { - return Ok(()); - } - sleep(Duration::from_millis(100)).await; - } - Err(miette::miette!("timed out waiting for listener {}", addr)) -} - pub(super) fn endpoint_returns_http_response_blocking( addr: SocketAddr, timeout: Duration, @@ -872,12 +812,6 @@ pub(super) fn endpoint_accepts_stable_connection_blocking( Ok(false) } -pub(super) fn find_header_end(buf: &[u8]) -> Option { - buf.windows(4) - .position(|window| window == b"\r\n\r\n") - .map(|index| index + 4) -} - pub(super) fn pid_is_alive(pid: u32) -> bool { #[cfg(unix)] { @@ -920,13 +854,6 @@ fn parse_process_status_code(raw: &str) -> Option { .map(|state| state.to_ascii_uppercase()) } -#[cfg(unix)] -pub(super) fn send_sigterm(pid: u32) { - unsafe { - libc::kill(pid as i32, libc::SIGTERM); - } -} - #[cfg(not(unix))] pub(super) fn send_sigterm(_pid: u32) {} @@ -952,141 +879,88 @@ pub(super) fn send_signal_to_process_group(root_pid: u32, signal: i32) { #[cfg(not(unix))] pub(super) fn send_signal_to_process_group(_root_pid: u32, _signal: i32) {} -pub(super) fn process_tree_postorder(root_pid: u32) -> Result> { - Ok(vec![root_pid]) +#[cfg(unix)] +fn parse_process_tree_line(raw: &str) -> Option<(u32, u32)> { + let mut fields = raw.split_ascii_whitespace(); + let pid = fields.next()?.parse::().ok()?; + let ppid = fields.next()?.parse::().ok()?; + Some((pid, ppid)) } -pub(super) async fn terminate_recorded_processes(root_pids: &[u32]) -> Result<()> { - for pid in root_pids { - send_sigterm(*pid); - if !wait_for_pid_exit(*pid, PROCESS_SHUTDOWN_GRACE_PERIOD).await { - #[cfg(unix)] - unsafe { - libc::kill(*pid as i32, libc::SIGKILL); +#[cfg(unix)] +fn process_tree_postorder_from_ps(raw: &str, root_pid: u32) -> Vec { + fn visit( + pid: u32, + children_by_parent: &std::collections::BTreeMap>, + seen: &mut std::collections::BTreeSet, + out: &mut Vec, + ) { + if !seen.insert(pid) { + return; + } + if let Some(children) = children_by_parent.get(&pid) { + for child in children { + visit(*child, children_by_parent, seen, out); } } + out.push(pid); } - Ok(()) -} -pub(crate) async fn resolve_link_external_url_for_output( - provider: &LaunchedSite, - provider_output_dir: &Path, - link: &RunLink, - consumer_kind: SiteKind, - run_root: &Path, - bridge_proxies: &mut BTreeMap, -) -> Result { - if !link_needs_bridge_proxy(provider.receipt.kind, consumer_kind) { - return external_slot_url(provider, provider_output_dir, link, consumer_kind); + let mut children_by_parent = std::collections::BTreeMap::>::new(); + for line in raw.lines() { + let Some((pid, ppid)) = parse_process_tree_line(line) else { + continue; + }; + children_by_parent.entry(ppid).or_default().push(pid); + } + for children in children_by_parent.values_mut() { + children.sort_unstable(); } - let port = ensure_bridge_proxy( - run_root, - provider, - provider_output_dir, - &link.export_name, - consumer_kind, - bridge_proxies, - ) - .await?; - bridge_proxy_external_url(port, link.protocol, consumer_kind) -} - -pub(super) fn link_needs_bridge_proxy(provider_kind: SiteKind, consumer_kind: SiteKind) -> bool { - matches!(consumer_kind, SiteKind::Compose | SiteKind::Kubernetes) - && provider_kind != SiteKind::Kubernetes + let mut out = Vec::new(); + visit( + root_pid, + &children_by_parent, + &mut std::collections::BTreeSet::new(), + &mut out, + ); + out } -pub(super) async fn ensure_bridge_proxy( - run_root: &Path, - provider: &LaunchedSite, - provider_output_dir: &Path, - export_name: &str, - consumer_kind: SiteKind, - bridge_proxies: &mut BTreeMap, -) -> Result { - let key = BridgeProxyKey { - provider_output_dir: provider_output_dir.display().to_string(), - export_name: export_name.to_string(), - consumer_kind, - }; - if let Some(proxy) = bridge_proxies.get_mut(&key) - && proxy.child.try_wait().into_diagnostic()?.is_none() +pub(super) fn process_tree_postorder(root_pid: u32) -> Result> { + #[cfg(unix)] { - return Ok(proxy.listen.port()); + let output = Command::new("ps") + .args(["-axo", "pid=,ppid="]) + .output() + .into_diagnostic() + .wrap_err("failed to query process tree via `ps`")?; + if !output.status.success() { + return Err(miette::miette!( + "`ps -axo pid=,ppid=` failed with status {}", + output.status + )); + } + Ok(process_tree_postorder_from_ps( + &String::from_utf8_lossy(&output.stdout), + root_pid, + )) } - let listen = bridge_proxy_bind_addr(consumer_kind, reserve_loopback_port()?); - let child = spawn_bridge_proxy(run_root, provider, provider_output_dir, export_name, listen)?; - wait_for_socket_listener(bridge_proxy_probe_addr(listen)).await?; - bridge_proxies.insert(key, BridgeProxyHandle { child, listen }); - Ok(listen.port()) + #[cfg(not(unix))] + { + Ok(vec![root_pid]) + } } -pub(super) fn spawn_bridge_proxy( - run_root: &Path, +pub(crate) async fn resolve_link_external_url_for_output( provider: &LaunchedSite, provider_output_dir: &Path, - export_name: &str, - listen: SocketAddr, -) -> Result { - let logs_root = run_root.join("bridge-proxies"); - fs::create_dir_all(&logs_root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", logs_root.display()))?; - let log_path = logs_root.join(format!("{export_name}.log")); - spawn_detached_child(run_root, &log_path, |cmd| { - cmd.arg("proxy") - .arg(provider_output_dir) - .arg("--export") - .arg(bridge_proxy_export_binding(export_name, listen)); - if provider.receipt.kind == SiteKind::Kubernetes { - let control = provider.router_control.to_string(); - cmd.arg("--router-addr") - .arg(provider.router_addr.to_string()) - .arg("--router-control-addr") - .arg(control); - } - }) -} - -pub(super) fn bridge_proxy_export_binding(export_name: &str, listen: SocketAddr) -> String { - format!("{export_name}={}:{}", listen.ip(), listen.port()) -} - -pub(super) fn bridge_proxy_bind_addr(consumer_kind: SiteKind, port: u16) -> SocketAddr { - host_service_bind_addr_for_consumer(consumer_kind, port) -} - -pub(super) fn bridge_proxy_probe_addr(listen: SocketAddr) -> SocketAddr { - listener_probe_addr(listen) -} - -pub(super) fn bridge_proxy_external_url( - port: u16, - protocol: NetworkProtocol, + link: &RunLink, consumer_kind: SiteKind, + _run_root: &Path, ) -> Result { - let host = host_service_host_for_consumer(consumer_kind); - Ok(match protocol { - NetworkProtocol::Http | NetworkProtocol::Https => format!("http://{host}:{port}"), - NetworkProtocol::Tcp => format!("tcp://{host}:{port}"), - _ => { - return Err(miette::miette!( - "mixed-site bridge proxy does not support protocol `{protocol}`" - )); - } - }) -} - -pub(crate) fn host_service_host_for_consumer(consumer_kind: SiteKind) -> String { - match consumer_kind { - SiteKind::Compose => CONTAINER_HOST_ALIAS.to_string(), - SiteKind::Direct | SiteKind::Vm | SiteKind::Kubernetes => { - container_host_for_consumer(SiteKind::Direct, consumer_kind) - } - } + external_slot_url(provider, provider_output_dir, link, consumer_kind) } pub fn site_controller_peer_router_url(controller_site_kind: SiteKind, route_port: u16) -> String { @@ -1098,7 +972,10 @@ pub fn site_controller_peer_router_url(controller_site_kind: SiteKind, route_por } pub(super) fn consumer_needs_host_wide_listener(consumer_kind: SiteKind) -> bool { - matches!(consumer_kind, SiteKind::Compose | SiteKind::Kubernetes) + matches!( + consumer_kind, + SiteKind::Compose | SiteKind::Kubernetes | SiteKind::Vm + ) || (cfg!(target_os = "linux") && matches!(consumer_kind, SiteKind::Direct)) } pub fn host_service_bind_addr_for_consumer(consumer_kind: SiteKind, port: u16) -> SocketAddr { @@ -1109,6 +986,27 @@ pub fn router_mesh_addr_for_consumer( provider_kind: SiteKind, consumer_kind: SiteKind, router_mesh_addr: &str, +) -> Result { + match consumer_kind { + SiteKind::Compose | SiteKind::Kubernetes => { + let addr = router_mesh_addr + .parse::() + .into_diagnostic() + .wrap_err_with(|| { + format!("invalid live router mesh address `{router_mesh_addr}`") + })?; + let host = container_host_for_consumer(provider_kind, consumer_kind); + Ok(format!("{host}:{}", addr.port())) + } + SiteKind::Direct => Ok(router_mesh_addr.to_string()), + SiteKind::Vm => Ok(router_mesh_addr.to_string()), + } +} + +pub fn router_mesh_addr_for_component_consumer( + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, ) -> Result { match consumer_kind { SiteKind::Compose | SiteKind::Kubernetes => { @@ -1142,23 +1040,31 @@ pub(super) fn host_proxy_bind_addr(needs_host_wide_listener: bool, port: u16) -> } } -pub(super) fn listener_probe_addr(listen: SocketAddr) -> SocketAddr { - if listen.ip().is_unspecified() { - SocketAddr::from(([127, 0, 0, 1], listen.port())) - } else { - listen - } -} - pub(super) fn external_slot_url( provider: &LaunchedSite, provider_output_dir: &Path, link: &RunLink, consumer_kind: SiteKind, ) -> Result { - let host = container_host_for_consumer(provider.receipt.kind, consumer_kind); + let router_mesh_addr = if let Some(router_mesh_addr) = + crate::runtime_api::published_router_mesh_addr_for_consumer_kind( + &provider.receipt, + consumer_kind, + ) { + router_mesh_addr.to_string() + } else { + router_mesh_addr_for_consumer( + provider.receipt.kind, + consumer_kind, + provider + .receipt + .router_mesh_addr + .as_deref() + .ok_or_else(|| miette::miette!("provider site is missing router mesh addr"))?, + )? + }; let route_id = provider_export_route_id(provider_output_dir, link)?; - let mut mesh_url = Url::parse(&format!("mesh://{}:{}", host, provider.router_addr.port())) + let mut mesh_url = Url::parse(&format!("mesh://{router_mesh_addr}")) .into_diagnostic() .wrap_err("failed to build mesh link url")?; let peer_key = @@ -1198,21 +1104,13 @@ pub(super) fn container_host_for_consumer( } pub(super) fn container_host_from_resolved_ip( - provider_kind: SiteKind, + _provider_kind: SiteKind, consumer_kind: SiteKind, container_host_ip: Option<&str>, ) -> String { match consumer_kind { SiteKind::Direct | SiteKind::Vm => "127.0.0.1".to_string(), - SiteKind::Compose => { - if provider_kind == SiteKind::Kubernetes { - container_host_ip - .unwrap_or(CONTAINER_HOST_ALIAS) - .to_string() - } else { - CONTAINER_HOST_ALIAS.to_string() - } - } + SiteKind::Compose => CONTAINER_HOST_ALIAS.to_string(), SiteKind::Kubernetes => container_host_ip .unwrap_or(CONTAINER_HOST_ALIAS) .to_string(), @@ -1274,16 +1172,25 @@ pub(super) fn resolve_desktop_container_host_ip() -> Option { #[cfg(test)] mod tests { - use std::collections::BTreeMap; + use std::{collections::BTreeMap, fs, net::SocketAddr}; - use amber_compiler::run_plan::SiteKind; + use amber_compiler::{ + mesh::{PROXY_METADATA_FILENAME, PROXY_METADATA_VERSION}, + run_plan::SiteKind, + }; + use amber_manifest::NetworkProtocol; + use amber_mesh::{MeshIdentity, MeshIdentityPublic, router_export_route_id}; + use amber_proxy::ControlEndpoint; + use base64::Engine as _; + use tempfile::tempdir; - #[cfg(unix)] - use super::parse_process_status_code; use super::{ - SITE_PLAN_SCHEMA, SITE_PLAN_VERSION, SiteSupervisorPlan, + LaunchedSite, SITE_PLAN_SCHEMA, SITE_PLAN_VERSION, SiteReceipt, SiteSupervisorPlan, + external_slot_url, resolve_link_external_url_for_output, router_mesh_addr_for_consumer, should_prepare_kubernetes_namespace, }; + #[cfg(unix)] + use super::{parse_process_status_code, process_tree_postorder_from_ps}; fn kubernetes_supervisor_plan(context: Option<&str>) -> SiteSupervisorPlan { SiteSupervisorPlan { @@ -1320,6 +1227,24 @@ mod tests { assert_eq!(parse_process_status_code(""), None); } + #[cfg(unix)] + #[test] + fn process_tree_postorder_from_ps_includes_descendants_before_root() { + let tree = process_tree_postorder_from_ps( + "\ +1 0 +42 1 +7 42 +8 42 +9 8 +100 1 +", + 42, + ); + + assert_eq!(tree, vec![7, 9, 8, 42]); + } + #[test] fn in_cluster_kubernetes_controller_skips_namespace_bootstrap() { let plan = kubernetes_supervisor_plan(None); @@ -1338,4 +1263,287 @@ mod tests { artifacts" ); } + + #[test] + fn direct_consumers_keep_router_mesh_addr_verbatim() { + let peer_addr = + router_mesh_addr_for_consumer(SiteKind::Direct, SiteKind::Direct, "127.0.0.1:24077") + .expect("direct consumers should keep direct peer mesh addresses"); + assert_eq!(peer_addr, "127.0.0.1:24077"); + } + + #[test] + fn vm_site_consumers_keep_router_mesh_addr_verbatim() { + let peer_addr = + router_mesh_addr_for_consumer(SiteKind::Direct, SiteKind::Vm, "127.0.0.1:24077") + .expect("vm site consumers should keep peer router mesh addresses"); + assert_eq!(peer_addr, "127.0.0.1:24077"); + } + + #[cfg(target_os = "linux")] + #[test] + fn direct_component_consumers_rewrite_loopback_router_mesh_addr_to_slirp_gateway() { + let peer_addr = super::router_mesh_addr_for_component_consumer( + SiteKind::Direct, + SiteKind::Direct, + "127.0.0.1:24077", + ) + .expect("direct component consumers should rewrite loopback peer mesh addresses"); + assert_eq!(peer_addr, "10.0.2.2:24077"); + } + + #[test] + fn host_service_bind_addr_matches_component_reachability() { + assert_eq!( + super::host_service_bind_addr_for_consumer(SiteKind::Compose, 24077), + SocketAddr::from(([0, 0, 0, 0], 24077)) + ); + assert_eq!( + super::host_service_bind_addr_for_consumer(SiteKind::Kubernetes, 24078), + SocketAddr::from(([0, 0, 0, 0], 24078)) + ); + assert_eq!( + super::host_service_bind_addr_for_consumer(SiteKind::Vm, 24079), + SocketAddr::from(([0, 0, 0, 0], 24079)) + ); + + let direct = super::host_service_bind_addr_for_consumer(SiteKind::Direct, 24080); + if cfg!(target_os = "linux") { + assert_eq!(direct, SocketAddr::from(([0, 0, 0, 0], 24080))); + } else { + assert_eq!(direct, SocketAddr::from(([127, 0, 0, 1], 24080))); + } + } + + #[test] + fn compose_consumers_reach_kubernetes_peers_via_host_alias() { + let peer_addr = super::router_mesh_addr_for_consumer( + SiteKind::Kubernetes, + SiteKind::Compose, + "127.0.0.1:24077", + ) + .expect("compose consumers should rewrite kubernetes peers to a host-reachable address"); + assert_eq!(peer_addr, "host.docker.internal:24077"); + } + + #[test] + fn kubernetes_consumers_resolve_cross_site_links_as_mesh_router_urls() { + let temp = tempfile::tempdir().expect("tempdir should create"); + fs::write( + temp.path().join(PROXY_METADATA_FILENAME), + serde_json::to_vec_pretty(&serde_json::json!({ + "version": PROXY_METADATA_VERSION, + "exports": { + "compose_http": { + "component": "/job-kind/compose_helper", + "provide": "http", + "capability_kind": "http", + "protocol": "http", + "router_mesh_port": 24000, + "route_id": router_export_route_id("compose_http", amber_mesh::MeshProtocol::Http), + } + } + })) + .expect("proxy metadata should serialize"), + ) + .expect("proxy metadata should write"); + + let router_identity = MeshIdentity::generate("/site/compose_local/router", None); + let provider = LaunchedSite { + receipt: SiteReceipt { + kind: SiteKind::Compose, + artifact_dir: temp.path().display().to_string(), + supervisor_pid: 0, + process_pid: None, + compose_project: Some("compose-test".to_string()), + kubernetes_namespace: None, + port_forward_pid: None, + context: None, + router_control: None, + router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: None, + kubernetes_consumer_router_mesh_addr: None, + router_identity_id: Some(router_identity.id.clone()), + router_public_key_b64: None, + site_controller_pid: None, + site_controller_url: None, + }, + router_control: ControlEndpoint::Tcp("127.0.0.1:24100".to_string()), + router_identity: MeshIdentityPublic::from_identity(&router_identity), + router_addr: SocketAddr::from(([127, 0, 0, 1], 24000)), + }; + let link = amber_compiler::run_plan::RunLink { + provider_site: "compose_local".to_string(), + consumer_site: "kind_local".to_string(), + provider_component: "/job-kind/compose_helper".to_string(), + provide: "http".to_string(), + consumer_component: "/job-kind/root".to_string(), + slot: "compose".to_string(), + weak: false, + protocol: NetworkProtocol::Http, + export_name: "compose_http".to_string(), + external_slot_name: "amber_link_compose_http".to_string(), + }; + + let resolved = tokio::runtime::Runtime::new() + .expect("tokio runtime should create") + .block_on(resolve_link_external_url_for_output( + &provider, + temp.path(), + &link, + SiteKind::Kubernetes, + temp.path(), + )) + .expect("kubernetes consumer link should resolve"); + + assert!( + resolved.starts_with("mesh://"), + "cross-site kubernetes consumers should route through mesh, got {resolved}" + ); + assert!( + !resolved.starts_with("http://"), + "cross-site kubernetes consumers should not bounce through an HTTP bridge proxy: \ + {resolved}" + ); + assert!(resolved.contains("peer_id=%2Fsite%2Fcompose_local%2Frouter")); + } + + #[test] + fn external_slot_url_prefers_host_published_mesh_addr_for_kubernetes_consumers() { + let temp = tempfile::tempdir().expect("tempdir should create"); + fs::write( + temp.path().join(PROXY_METADATA_FILENAME), + serde_json::to_vec_pretty(&serde_json::json!({ + "version": PROXY_METADATA_VERSION, + "exports": { + "compose_http": { + "component": "/job-kind/compose_helper", + "provide": "http", + "capability_kind": "http", + "protocol": "http", + "router_mesh_port": 24000, + "route_id": router_export_route_id("compose_http", amber_mesh::MeshProtocol::Http), + } + } + })) + .expect("proxy metadata should serialize"), + ) + .expect("proxy metadata should write"); + let router_identity = MeshIdentity::generate("/site/compose_local/router", None); + let provider = LaunchedSite { + receipt: SiteReceipt { + kind: SiteKind::Compose, + artifact_dir: temp.path().display().to_string(), + supervisor_pid: 0, + process_pid: None, + compose_project: Some("compose-test".to_string()), + kubernetes_namespace: None, + port_forward_pid: None, + context: None, + router_control: Some("unix:///tmp/router.sock".to_string()), + router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: Some("host.docker.internal:24000".to_string()), + kubernetes_consumer_router_mesh_addr: Some("192.168.65.254:24000".to_string()), + router_identity_id: Some(router_identity.id.clone()), + router_public_key_b64: Some( + base64::engine::general_purpose::STANDARD.encode(router_identity.public_key), + ), + site_controller_pid: None, + site_controller_url: None, + }, + router_control: ControlEndpoint::Tcp("127.0.0.1:24100".to_string()), + router_identity: MeshIdentityPublic::from_identity(&router_identity), + router_addr: SocketAddr::from(([127, 0, 0, 1], 24000)), + }; + let link = amber_compiler::run_plan::RunLink { + provider_site: "compose_local".to_string(), + consumer_site: "kind_local".to_string(), + provider_component: "/job-kind/compose_helper".to_string(), + provide: "http".to_string(), + consumer_component: "/job-kind/root".to_string(), + slot: "compose".to_string(), + weak: false, + protocol: NetworkProtocol::Http, + export_name: "compose_http".to_string(), + external_slot_name: "amber_link_compose_http".to_string(), + }; + + let resolved = external_slot_url(&provider, temp.path(), &link, SiteKind::Kubernetes) + .expect("external slot url should resolve"); + + assert!( + resolved.starts_with("mesh://192.168.65.254:24000"), + "container consumers should use the host-published mesh address, got {resolved}" + ); + } + + #[test] + fn external_slot_url_keeps_vm_router_mesh_addr_loopback() { + let temp = tempdir().expect("tempdir"); + fs::write( + temp.path().join(PROXY_METADATA_FILENAME), + serde_json::to_vec_pretty(&serde_json::json!({ + "version": PROXY_METADATA_VERSION, + "exports": { + "compose_http": { + "component": "/job-vm/compose_helper", + "provide": "http", + "capability_kind": "http", + "protocol": "http", + "router_mesh_port": 24000, + "route_id": router_export_route_id("compose_http", amber_mesh::MeshProtocol::Http), + } + } + })) + .expect("proxy metadata should serialize"), + ) + .expect("proxy metadata should write"); + let router_identity = MeshIdentity::generate("/site/compose_local/router", None); + let provider = LaunchedSite { + receipt: SiteReceipt { + kind: SiteKind::Compose, + artifact_dir: temp.path().display().to_string(), + supervisor_pid: 1, + process_pid: None, + compose_project: None, + kubernetes_namespace: None, + port_forward_pid: None, + context: None, + router_control: Some("unix:///tmp/router.sock".to_string()), + router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: Some("host.docker.internal:24000".to_string()), + kubernetes_consumer_router_mesh_addr: Some("192.168.65.254:24000".to_string()), + router_identity_id: Some(router_identity.id.clone()), + router_public_key_b64: Some( + base64::engine::general_purpose::STANDARD.encode(router_identity.public_key), + ), + site_controller_pid: None, + site_controller_url: None, + }, + router_control: ControlEndpoint::Tcp("127.0.0.1:24100".to_string()), + router_identity: MeshIdentityPublic::from_identity(&router_identity), + router_addr: SocketAddr::from(([127, 0, 0, 1], 24000)), + }; + let link = amber_compiler::run_plan::RunLink { + provider_site: "compose_local".to_string(), + consumer_site: "vm_local".to_string(), + provider_component: "/job-vm/compose_helper".to_string(), + provide: "http".to_string(), + consumer_component: "/job-vm/root".to_string(), + slot: "compose".to_string(), + weak: false, + protocol: NetworkProtocol::Http, + export_name: "compose_http".to_string(), + external_slot_name: "amber_link_compose_http".to_string(), + }; + + let resolved = external_slot_url(&provider, temp.path(), &link, SiteKind::Vm).expect( + "external slot url should resolve for vm site consumers without slirp rewriting", + ); + + assert!( + resolved.starts_with("mesh://127.0.0.1:24000"), + "vm site consumers should use the live router mesh address, got {resolved}" + ); + } } diff --git a/runtime/site-controller/src/orchestration.rs b/runtime/site-controller/src/orchestration.rs index 53fb35c6..2e8dd4ba 100644 --- a/runtime/site-controller/src/orchestration.rs +++ b/runtime/site-controller/src/orchestration.rs @@ -1,9 +1,5 @@ -#[cfg(target_os = "linux")] -use std::net::Ipv4Addr; - use amber_mesh::{ - InboundRoute, InboundTarget, MeshConfigPublic, MeshPeer, OutboundRoute, - router_external_route_id, + InboundRoute, InboundTarget, MeshConfigPublic, MeshPeer, router_external_route_id, }; use amber_proxy::{ControlEndpoint, fetch_router_identity}; @@ -93,6 +89,7 @@ pub(crate) struct ResolveExternalLinkUrlRequest { pub(crate) child_id: u64, pub(crate) link: RunLink, pub(crate) consumer_kind: SiteKind, + pub(crate) provider_in_child: bool, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -185,6 +182,8 @@ pub(super) fn site_receipt_from_manager_state(state: &SiteManagerStateView) -> S context: state.context.clone(), router_control: state.router_control.clone(), router_mesh_addr: state.router_mesh_addr.clone(), + compose_consumer_router_mesh_addr: state.compose_consumer_router_mesh_addr.clone(), + kubernetes_consumer_router_mesh_addr: state.kubernetes_consumer_router_mesh_addr.clone(), router_identity_id: state.router_identity_id.clone(), router_public_key_b64: state.router_public_key_b64.clone(), site_controller_pid: state.site_controller_pid, @@ -196,6 +195,9 @@ pub(super) fn load_site_manager_state( app: &ControlStateApp, site_id: &str, ) -> std::result::Result { + if site_id == app.controller_plan.site_id { + return Ok(local_site_manager_state_view(app)); + } let state_path = site_state_path(&app.state_root, site_id); if state_path.is_file() { return read_json(&state_path, "site manager state").map_err(|err| { @@ -205,9 +207,6 @@ pub(super) fn load_site_manager_state( ) }); } - if site_id == app.controller_plan.site_id { - return Ok(local_site_manager_state_view(app)); - } Err(protocol_error( ProtocolErrorCode::SiteNotActive, &format!( @@ -219,6 +218,8 @@ pub(super) fn load_site_manager_state( fn local_site_manager_state_view(app: &ControlStateApp) -> SiteManagerStateView { let runtime_plan = site_controller_runtime_plan_from_controller_plan(&app.controller_plan); + let persisted_state = + load_site_manager_state_at(Path::new(&app.controller_plan.site_state_root)).ok(); let router_identity = app .runtime .load_live_site_router_mesh_config(&runtime_plan) @@ -239,10 +240,57 @@ fn local_site_manager_state_view(app: &ControlStateApp) -> SiteManagerStateView kubernetes_namespace: app.controller_plan.kubernetes_namespace.clone(), port_forward_pid: None, context: app.controller_plan.context.clone(), - router_control: app.controller_plan.local_router_control.clone(), - router_mesh_addr: app.controller_plan.published_router_mesh_addr.clone(), - router_identity_id: router_identity.as_ref().map(|(id, _)| id.clone()), - router_public_key_b64: router_identity.map(|(_, public_key_b64)| public_key_b64), + router_control: app + .controller_plan + .local_router_control + .clone() + .or_else(|| { + persisted_state + .as_ref() + .and_then(|state| state.router_control.clone()) + }), + router_mesh_addr: app + .controller_plan + .published_router_mesh_addr + .clone() + .or_else(|| { + persisted_state + .as_ref() + .and_then(|state| state.router_mesh_addr.clone()) + }), + compose_consumer_router_mesh_addr: app + .controller_plan + .compose_consumer_router_mesh_addr + .clone() + .or_else(|| { + persisted_state + .as_ref() + .and_then(|state| state.compose_consumer_router_mesh_addr.clone()) + }), + kubernetes_consumer_router_mesh_addr: app + .controller_plan + .kubernetes_consumer_router_mesh_addr + .clone() + .or_else(|| { + persisted_state + .as_ref() + .and_then(|state| state.kubernetes_consumer_router_mesh_addr.clone()) + }), + router_identity_id: router_identity + .as_ref() + .map(|(id, _)| id.clone()) + .or_else(|| { + persisted_state + .as_ref() + .and_then(|state| state.router_identity_id.clone()) + }), + router_public_key_b64: router_identity + .map(|(_, public_key_b64)| public_key_b64) + .or_else(|| { + persisted_state + .as_ref() + .and_then(|state| state.router_public_key_b64.clone()) + }), site_controller_pid: None, site_controller_url: Some(app.controller_plan.authority_url.clone()), } @@ -445,16 +493,12 @@ pub(super) async fn resolve_external_link_url_local( ), )); } - let child = { - let state = app.control_state.lock().await; - cloned_child_record(&state, request.child_id)? - }; let provider = load_launched_site(app, &app.controller_plan.site_id)?; - let provider_output_dir = provider_output_dir_for_link( + let provider_output_dir = provider_output_dir_for_request( app, - &child, + request.child_id, Path::new(&provider.receipt.artifact_dir), - &request.link, + request.provider_in_child, ); let external_url = app .runtime @@ -1002,7 +1046,6 @@ pub(super) fn dynamic_capability_origin_route_surface( runtime: &LiveComponentRuntimeMetadata, site_components: &BTreeMap, site_router: &MeshConfigPublic, - site_kind: SiteKind, route_id: &str, root_authority_selector: &RootAuthoritySelectorIr, allowed_issuers: Vec, @@ -1034,7 +1077,7 @@ pub(super) fn dynamic_capability_origin_route_surface( protocol: static_route.protocol, http_plugins: static_route.http_plugins.clone(), target: InboundTarget::MeshForward { - peer_addr: runtime.host_mesh_addr.clone(), + peer_addr: runtime.router_reachable_mesh_addr.clone(), peer_id: runtime.mesh_config.identity.id.clone(), route_id: static_route.route_id.clone(), capability: static_route.capability.clone(), @@ -1084,8 +1127,15 @@ pub(super) fn dynamic_capability_origin_route_surface( ), })? .expect("self-provide roots should be handled before outbound route resolution"); - let peer_addr = - dynamic_capability_origin_mesh_peer_addr(site_components, site_kind, route); + let peer_addr = if route.peer_id == runtime.mesh_config.identity.id { + runtime.router_reachable_mesh_addr.clone() + } else { + site_components + .values() + .find(|component| component.mesh_config.identity.id == route.peer_id) + .map(|component| component.router_reachable_mesh_addr.clone()) + .unwrap_or_else(|| route.peer_addr.clone()) + }; Ok(( InboundRoute { route_id: route_id.to_string(), @@ -1139,28 +1189,6 @@ pub(super) fn dynamic_capability_origin_route_surface( } } -fn dynamic_capability_origin_mesh_peer_addr( - site_components: &BTreeMap, - site_kind: SiteKind, - route: &OutboundRoute, -) -> String { - if let Some(peer_runtime) = site_components - .values() - .find(|component| component.mesh_config.identity.id == route.peer_id) - { - return peer_runtime.host_mesh_addr.clone(); - } - if matches!(site_kind, SiteKind::Direct | SiteKind::Vm) { - #[cfg(target_os = "linux")] - if let Ok(addr) = route.peer_addr.parse::() - && addr.ip() == Ipv4Addr::new(10, 0, 2, 2) - { - return SocketAddr::from((Ipv4Addr::LOCALHOST, addr.port())).to_string(); - } - } - route.peer_addr.clone() -} - pub(super) fn dynamic_capability_origin_target_mesh_peer( runtime: &LiveComponentRuntimeMetadata, site_components: &BTreeMap, @@ -1311,7 +1339,6 @@ pub(super) async fn publish_dynamic_capability_origin_local( &runtime, &site_components, &site_router, - site_plan.kind, &request.route_id, &request.root_authority_selector, allowed_issuers, @@ -1571,6 +1598,7 @@ pub(super) async fn publish_external_slot_overlay( child_id: child.child_id, link: link.clone(), consumer_kind, + provider_in_child: provider_in_child_for_link(child, link), }; let external_url = if link.provider_site == app.controller_plan.site_id { resolve_external_link_url_local(app, &resolve_request) @@ -1690,25 +1718,28 @@ pub(super) fn link_still_required( }) } -pub(super) fn provider_output_dir_for_link( +pub(super) fn provider_output_dir_for_request( app: &ControlStateApp, - child: &LiveChildRecord, + child_id: u64, provider_artifact_dir: &Path, - link: &RunLink, + provider_in_child: bool, ) -> PathBuf { - let provider_in_child = child.fragment.as_ref().is_some_and(|fragment| { - fragment - .components - .iter() - .any(|component| component.moniker == link.provider_component) - }); if !provider_in_child { return provider_artifact_dir.to_path_buf(); } - site_controller_runtime_child_root_for_site(local_site_state_root(app), child.child_id) + site_controller_runtime_child_root_for_site(local_site_state_root(app), child_id) .join("artifact") } +pub(super) fn provider_in_child_for_link(child: &LiveChildRecord, link: &RunLink) -> bool { + child.fragment.as_ref().is_some_and(|fragment| { + fragment + .components + .iter() + .any(|component| component.moniker == link.provider_component) + }) +} + pub(super) fn export_peer_route_id( child: &LiveChildRecord, link: &RunLink, @@ -1722,12 +1753,7 @@ pub(super) fn export_peer_route_id( ), ) })?; - let provider_in_child = child.fragment.as_ref().is_some_and(|fragment| { - fragment - .components - .iter() - .any(|component| component.moniker == link.provider_component) - }); + let provider_in_child = provider_in_child_for_link(child, link); Ok(if provider_in_child { router_dynamic_export_route_id(&link.provider_component, &link.export_name, protocol) } else { diff --git a/runtime/site-controller/src/planner.rs b/runtime/site-controller/src/planner.rs index 694eb36a..b8cd886f 100644 --- a/runtime/site-controller/src/planner.rs +++ b/runtime/site-controller/src/planner.rs @@ -3307,6 +3307,10 @@ pub(super) struct SiteManagerStateView { #[serde(default)] pub(super) router_mesh_addr: Option, #[serde(default)] + pub(super) compose_consumer_router_mesh_addr: Option, + #[serde(default)] + pub(super) kubernetes_consumer_router_mesh_addr: Option, + #[serde(default)] pub(super) router_identity_id: Option, #[serde(default)] pub(super) router_public_key_b64: Option, diff --git a/runtime/site-controller/src/runtime_api.rs b/runtime/site-controller/src/runtime_api.rs index f2d7b982..306dd132 100644 --- a/runtime/site-controller/src/runtime_api.rs +++ b/runtime/site-controller/src/runtime_api.rs @@ -83,6 +83,15 @@ pub(crate) trait SiteControllerRuntime: Send + Sync { router_mesh_addr: &str, ) -> Result; + fn router_mesh_addr_for_component_consumer( + &self, + provider_kind: SiteKind, + consumer_kind: SiteKind, + router_mesh_addr: &str, + ) -> Result { + self.router_mesh_addr_for_consumer(provider_kind, consumer_kind, router_mesh_addr) + } + fn update_desired_overlay_for_consumer( &self, site_state_root: &Path, @@ -132,6 +141,10 @@ pub struct SiteReceipt { #[serde(default, skip_serializing_if = "Option::is_none")] pub router_mesh_addr: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub compose_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub kubernetes_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub router_identity_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub router_public_key_b64: Option, @@ -177,7 +190,8 @@ pub struct SiteControllerRuntimePlan { #[derive(Clone, Debug)] pub struct LiveComponentRuntimeMetadata { pub moniker: String, - pub host_mesh_addr: String, + pub router_reachable_mesh_addr: String, + pub component_reachable_mesh_addr: String, pub control_endpoint: Option, pub mesh_config: MeshConfigPublic, } @@ -468,3 +482,14 @@ pub fn launched_site_from_receipt( router_addr, }) } + +pub fn published_router_mesh_addr_for_consumer_kind( + site_receipt: &SiteReceipt, + consumer_kind: SiteKind, +) -> Option<&str> { + match consumer_kind { + SiteKind::Compose => site_receipt.compose_consumer_router_mesh_addr.as_deref(), + SiteKind::Kubernetes => site_receipt.kubernetes_consumer_router_mesh_addr.as_deref(), + SiteKind::Direct | SiteKind::Vm => None, + } +} diff --git a/runtime/site-controller/src/site_controller.rs b/runtime/site-controller/src/site_controller.rs index a5ab3f53..a2ee5054 100644 --- a/runtime/site-controller/src/site_controller.rs +++ b/runtime/site-controller/src/site_controller.rs @@ -18,8 +18,7 @@ use super::{ ccs_api::{self, FrameworkComponentInspectRequest, FrameworkComponentInspectResponse}, control_state_api::{ self, DynamicCapsInspectRequest, DynamicCapsInspectResponse, DynamicCapsMutateRequest, - DynamicCapsMutateResponse, local_component_runtime, - resolve_dynamic_capability_origin_internal, + DynamicCapsMutateResponse, resolve_dynamic_capability_origin_internal, }, dynamic_caps::{ self, ControlDynamicGrantAuthoritySyncRequest, ControlDynamicGrantAuthoritySyncResponse, @@ -789,6 +788,7 @@ pub(super) async fn execute_site_controller_dynamic_caps_inspect( ); } held.sort_by(|left, right| left.held_id.cmp(&right.held_id)); + held.dedup_by(|left, right| left.held_id == right.held_id); Ok(DynamicCapsInspectResponse::HeldList(HeldListResponse { held, })) @@ -852,9 +852,13 @@ pub(super) async fn execute_site_controller_dynamic_caps_inspect( local_resolve_origin(app, request).await?, )); } - let holder_runtime = - local_component_runtime(&app.control, &state, &request.holder_component_id) - .map_err(ProtocolApiError::from)?; + let holder_peer = control_state_api::local_component_runtime( + &app.control, + &state, + &request.holder_component_id, + )? + .mesh_config + .identity; Ok(DynamicCapsInspectResponse::ResolveOrigin( peer_dynamic_caps_post_via_router( app, @@ -863,9 +867,9 @@ pub(super) async fn execute_site_controller_dynamic_caps_inspect( &InternalDynamicResolveOriginRequest { holder_component_id: request.holder_component_id, source: request.source, - holder_peer_id: holder_runtime.mesh_config.identity.id.clone(), + holder_peer_id: holder_peer.id, holder_peer_key_b64: base64::engine::general_purpose::STANDARD - .encode(holder_runtime.mesh_config.identity.public_key), + .encode(holder_peer.public_key), holder_site_kind: app.control.controller_plan.kind, }, ) diff --git a/runtime/site-controller/src/state.rs b/runtime/site-controller/src/state.rs index 79272bad..a232f97a 100644 --- a/runtime/site-controller/src/state.rs +++ b/runtime/site-controller/src/state.rs @@ -273,6 +273,10 @@ pub struct SiteControllerPlan { pub local_router_control: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub published_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub compose_consumer_router_mesh_addr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub kubernetes_consumer_router_mesh_addr: Option, pub state_path: String, pub run_root: String, pub state_root: String, @@ -1238,6 +1242,8 @@ pub fn write_site_controller_plan( peer_router_mesh_addrs: &BTreeMap, local_router_control: Option<&str>, published_router_mesh_addr: Option<&str>, + compose_consumer_router_mesh_addr: Option<&str>, + kubernetes_consumer_router_mesh_addr: Option<&str>, state_path: &Path, run_root: &Path, state_root: &Path, @@ -1269,6 +1275,9 @@ pub fn write_site_controller_plan( peer_router_mesh_addrs: peer_router_mesh_addrs.clone(), local_router_control: local_router_control.map(str::to_string), published_router_mesh_addr: published_router_mesh_addr.map(str::to_string), + compose_consumer_router_mesh_addr: compose_consumer_router_mesh_addr.map(str::to_string), + kubernetes_consumer_router_mesh_addr: kubernetes_consumer_router_mesh_addr + .map(str::to_string), state_path: state_path.display().to_string(), run_root: run_root.display().to_string(), state_root: state_root.display().to_string(), diff --git a/runtime/site-controller/src/tests.rs b/runtime/site-controller/src/tests.rs index 2e0b8216..f4583a65 100644 --- a/runtime/site-controller/src/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -1,3 +1,5 @@ +#[cfg(all(unix, not(target_os = "macos")))] +use std::os::unix::fs::MetadataExt; use std::{ fs, future::Future, @@ -162,7 +164,8 @@ impl SiteControllerRuntime for TestSiteControllerRuntime { moniker.to_string(), LiveComponentRuntimeMetadata { moniker: moniker.to_string(), - host_mesh_addr: format!("127.0.0.1:{port}"), + router_reachable_mesh_addr: format!("127.0.0.1:{port}"), + component_reachable_mesh_addr: format!("127.0.0.1:{port}"), control_endpoint: None, mesh_config, }, @@ -1242,13 +1245,15 @@ async fn compile_dynamic_caps_binding_state() -> FrameworkControlState { fn test_live_component_runtime( moniker: &str, peer_id: &str, - host_mesh_addr: &str, + router_reachable_mesh_addr: &str, + component_reachable_mesh_addr: &str, inbound: Vec, outbound: Vec, ) -> LiveComponentRuntimeMetadata { LiveComponentRuntimeMetadata { moniker: moniker.to_string(), - host_mesh_addr: host_mesh_addr.to_string(), + router_reachable_mesh_addr: router_reachable_mesh_addr.to_string(), + component_reachable_mesh_addr: component_reachable_mesh_addr.to_string(), control_endpoint: None, mesh_config: MeshConfigPublic { identity: MeshIdentityPublic { @@ -1287,11 +1292,12 @@ fn test_live_site_router(inbound: Vec) -> MeshConfigPublic { } #[test] -fn dynamic_capability_origin_self_provide_routes_via_component_mesh() { +fn dynamic_capability_origin_self_provide_routes_via_router_reachable_mesh() { let runtime = test_live_component_runtime( "/provider", "/provider", "127.0.0.1:24001", + "10.0.2.2:24001", vec![InboundRoute { route_id: "provider-route".to_string(), capability: "provider.api".to_string(), @@ -1304,14 +1310,13 @@ fn dynamic_capability_origin_self_provide_routes_via_component_mesh() { }], Vec::new(), ); - let site_components = BTreeMap::from([(runtime.moniker.clone(), runtime.clone())]); let site_router = test_live_site_router(Vec::new()); + let site_components = BTreeMap::new(); let (route, capability, protocol) = dynamic_capability_origin_route_surface( &runtime, &site_components, &site_router, - SiteKind::Direct, "dynamic-origin", &RootAuthoritySelectorIr::SelfProvide { component_id: "/provider".to_string(), @@ -1343,6 +1348,7 @@ fn dynamic_capability_origin_binding_routes_same_site_provider_via_mesh() { "/consumer", "/consumer", "127.0.0.1:24002", + "10.0.2.2:24002", Vec::new(), vec![OutboundRoute { route_id: "provider-route".to_string(), @@ -1362,21 +1368,18 @@ fn dynamic_capability_origin_binding_routes_same_site_provider_via_mesh() { let provider_runtime = test_live_component_runtime( "/provider", "/provider", - "127.0.0.1:24001", + "127.0.0.1:24099", + "10.0.2.2:24099", Vec::new(), Vec::new(), ); - let site_components = BTreeMap::from([ - (holder_runtime.moniker.clone(), holder_runtime.clone()), - (provider_runtime.moniker.clone(), provider_runtime.clone()), - ]); let site_router = test_live_site_router(Vec::new()); + let site_components = BTreeMap::from([(provider_runtime.moniker.clone(), provider_runtime)]); let (route, capability, protocol) = dynamic_capability_origin_route_surface( &holder_runtime, &site_components, &site_router, - SiteKind::Direct, "dynamic-origin", &RootAuthoritySelectorIr::Binding { consumer_component_id: "components./consumer".to_string(), @@ -1393,7 +1396,7 @@ fn dynamic_capability_origin_binding_routes_same_site_provider_via_mesh() { assert_eq!( route.target, InboundTarget::MeshForward { - peer_addr: "127.0.0.1:24001".to_string(), + peer_addr: "127.0.0.1:24099".to_string(), peer_id: "/provider".to_string(), route_id: "provider-route".to_string(), capability: "provider.api".to_string(), @@ -1407,6 +1410,7 @@ fn dynamic_capability_origin_external_slot_routes_via_router_external_target() { "/consumer", "/consumer", "127.0.0.1:24002", + "10.0.2.2:24002", Vec::new(), vec![OutboundRoute { route_id: "router:external:catalog_api:http".to_string(), @@ -1423,7 +1427,6 @@ fn dynamic_capability_origin_external_slot_routes_via_router_external_target() { capability: "catalog_api".to_string(), }], ); - let site_components = BTreeMap::from([(runtime.moniker.clone(), runtime.clone())]); let site_router = test_live_site_router(vec![InboundRoute { route_id: "router:external:catalog_api:http".to_string(), capability: "catalog_api".to_string(), @@ -1437,12 +1440,12 @@ fn dynamic_capability_origin_external_slot_routes_via_router_external_target() { }, allowed_issuers: vec!["/consumer".to_string()], }]); + let site_components = BTreeMap::new(); let (route, capability, protocol) = dynamic_capability_origin_route_surface( &runtime, &site_components, &site_router, - SiteKind::Direct, "dynamic-origin", &RootAuthoritySelectorIr::ExternalSlotBinding { consumer_component_id: "components./consumer".to_string(), @@ -1467,62 +1470,13 @@ fn dynamic_capability_origin_external_slot_routes_via_router_external_target() { ); } -#[test] -fn dynamic_capability_origin_binding_rewrites_linux_slirp_peer_addr_for_host_router() { - let runtime = test_live_component_runtime( - "/consumer", - "/consumer", - "127.0.0.1:24002", - Vec::new(), - vec![OutboundRoute { - route_id: "remote-route".to_string(), - rewrite_route_id: None, - slot: "provider".to_string(), - capability_kind: Some("http".to_string()), - capability_profile: None, - listen_port: 20000, - listen_addr: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - peer_addr: "10.0.2.2:24077".to_string(), - peer_id: "/remote".to_string(), - capability: "provider.api".to_string(), - }], - ); - let site_components = BTreeMap::from([(runtime.moniker.clone(), runtime.clone())]); - let site_router = test_live_site_router(Vec::new()); - - let (route, _, _) = dynamic_capability_origin_route_surface( - &runtime, - &site_components, - &site_router, - SiteKind::Direct, - "dynamic-origin", - &RootAuthoritySelectorIr::Binding { - consumer_component_id: "components./consumer".to_string(), - slot_name: "provider".to_string(), - provider_component_id: "components./remote".to_string(), - provider_capability_name: "provider.api".to_string(), - }, - vec!["peer-consumer".to_string()], - ) - .expect("binding origin surface should resolve"); - - let InboundTarget::MeshForward { peer_addr, .. } = route.target else { - panic!("dynamic origin route should forward through mesh"); - }; - #[cfg(target_os = "linux")] - assert_eq!(peer_addr, "127.0.0.1:24077"); - #[cfg(not(target_os = "linux"))] - assert_eq!(peer_addr, "10.0.2.2:24077"); -} - #[test] fn dynamic_capability_origin_target_mesh_peer_uses_self_identity_for_self_provide() { let runtime = test_live_component_runtime( "/provider", "/provider", "127.0.0.1:24001", + "10.0.2.2:24001", vec![InboundRoute { route_id: "provider-route".to_string(), capability: "provider.api".to_string(), @@ -1541,7 +1495,6 @@ fn dynamic_capability_origin_target_mesh_peer_uses_self_identity_for_self_provid &runtime, &site_components, &site_router, - SiteKind::Direct, "dynamic-origin", &RootAuthoritySelectorIr::SelfProvide { component_id: "/provider".to_string(), @@ -1568,7 +1521,8 @@ fn dynamic_capability_origin_target_mesh_peer_uses_runtime_peer_catalog_for_bind }; let runtime = LiveComponentRuntimeMetadata { moniker: "/consumer".to_string(), - host_mesh_addr: "127.0.0.1:24002".to_string(), + router_reachable_mesh_addr: "127.0.0.1:24002".to_string(), + component_reachable_mesh_addr: "127.0.0.1:24002".to_string(), control_endpoint: None, mesh_config: MeshConfigPublic { identity: MeshIdentityPublic { @@ -1608,7 +1562,6 @@ fn dynamic_capability_origin_target_mesh_peer_uses_runtime_peer_catalog_for_bind &runtime, &site_components, &site_router, - SiteKind::Direct, "dynamic-origin", &RootAuthoritySelectorIr::Binding { consumer_component_id: "components./consumer".to_string(), @@ -2609,6 +2562,8 @@ fn test_control_state_app( peer_router_mesh_addrs: BTreeMap::new(), local_router_control: None, published_router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: Some("host.docker.internal:24000".to_string()), + kubernetes_consumer_router_mesh_addr: Some("192.168.65.254:24000".to_string()), state_path: state_path.display().to_string(), run_root: run_root.display().to_string(), state_root: state_root.display().to_string(), @@ -4073,6 +4028,7 @@ async fn install_dynamic_caps_origin_fixture(app: &ControlStateApp) -> tokio::ta "/provider", "/provider", "127.0.0.1:24001", + "10.0.2.2:24001", Vec::new(), Vec::new(), ) @@ -4085,6 +4041,7 @@ async fn install_dynamic_caps_origin_fixture(app: &ControlStateApp) -> tokio::ta "/alice", "/alice", "127.0.0.1:24002", + "10.0.2.2:24002", Vec::new(), vec![OutboundRoute { route_id: "provider-route".to_string(), @@ -4386,6 +4343,165 @@ async fn inspect_ref_routes_remote_grants_via_synced_authority_site() { ); } +#[tokio::test] +async fn held_list_aggregates_remote_grants_via_peer_router() { + let dir = TempDir::new().expect("temp dir"); + let base = compile_dynamic_caps_binding_state().await; + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "direct_a".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + ), + ( + "direct_b".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_a".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([ + ("/provider".to_string(), "direct_a".to_string()), + ("/alice".to_string(), "direct_a".to_string()), + ("/bob".to_string(), "direct_b".to_string()), + ]), + dynamic_capabilities: None, + framework_children: None, + }; + let mut authoritative = compile_control_state_from_ir_with_run_id( + base.base_scenario.clone(), + Some(&placement), + "test-run", + ) + .await; + let alice_root = super::dynamic_caps::source_key_from_held_id( + &authoritative, + "components./alice", + &root_held_id_for(&authoritative, "components./alice"), + ) + .expect("alice root source should resolve"); + let share = super::dynamic_caps::share_dynamic_capability( + &mut authoritative, + "components./alice", + &alice_root, + "components./bob", + None, + &json!({}), + ) + .expect("cross-site share should succeed"); + let grant_id = match share { + super::dynamic_caps::DynamicCapabilityShareOutcome::Created { grant_id, .. } => grant_id, + _ => panic!("cross-site share should create a grant"), + }; + + let mut holder_state = authoritative.clone(); + localize_framework_control_state(&mut holder_state, "direct_b") + .expect("holder site state should localize"); + let state_path = dir.path().join("control-state.json"); + write_control_state(&state_path, &holder_state).expect("holder state should write"); + + let hits = Arc::new(std::sync::Mutex::new(Vec::::new())); + let remote_grant_id = grant_id.clone(); + let router = Router::new().route( + "/v1/controller/dynamic-caps/held", + axum::routing::post({ + let hits = hits.clone(); + let remote_grant_id = remote_grant_id.clone(); + move |headers: HeaderMap, + Json(request): Json| { + let hits = hits.clone(); + let remote_grant_id = remote_grant_id.clone(); + async move { + assert_eq!( + headers + .get(super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER) + .and_then(|value| value.to_str().ok()), + Some("1"), + "peer-routed held_list should stay local on the destination controller", + ); + hits.lock() + .expect("held-list hit log poisoned") + .push(request.holder_component_id.clone()); + Json(amber_mesh::dynamic_caps::HeldListResponse { + held: vec![HeldEntrySummary { + held_id: super::dynamic_caps::held_id_for_grant(&remote_grant_id), + entry_kind: HeldEntryKind::DelegatedGrant, + grant_id: Some(remote_grant_id), + root_authority_selector: None, + state: HeldEntryState::Live, + from_component: Some("components./alice".to_string()), + descriptor: DescriptorIr { + kind: "http".to_string(), + label: "provider.http".to_string(), + profile: None, + }, + materializations: Vec::new(), + }], + }) + } + } + }), + ); + let (authority_base_url, _authority_handle) = spawn_test_router(router).await; + + let mut app = test_control_state_app(&dir, holder_state, state_path); + let controller_plan = Arc::make_mut(&mut app.controller_plan); + controller_plan.site_id = "direct_b".to_string(); + controller_plan.router_identity_id = "/site/direct_b/router".to_string(); + controller_plan.peer_site_router_urls = + BTreeMap::from([("direct_a".to_string(), authority_base_url)]); + let controller_app = SiteControllerApp { + control: app, + router_auth_token: Arc::::from("test-router-auth"), + ready: ready_site_controller_flag(), + }; + + let response = super::site_controller::execute_site_controller_dynamic_caps_inspect( + &controller_app, + super::control_state_api::DynamicCapsInspectRequest::HeldList( + dynamic_caps::ControlDynamicHeldListRequest { + holder_component_id: "components./bob".to_string(), + }, + ), + false, + ) + .await + .expect("holder site should aggregate remote grants through the authority site router"); + + let super::control_state_api::DynamicCapsInspectResponse::HeldList(response) = response else { + panic!("held_list should return a held list response"); + }; + assert!( + response + .held + .iter() + .any(|entry| entry.entry_kind == HeldEntryKind::RootAuthority), + "holder site should keep its local root authority entries", + ); + assert!( + response.held.iter().any(|entry| { + entry.entry_kind == HeldEntryKind::DelegatedGrant + && entry.grant_id.as_deref() == Some(grant_id.as_str()) + }), + "holder site should merge delegated grants from the authority site", + ); + assert_eq!( + hits.lock().expect("held-list hit log poisoned").as_slice(), + &["components./bob".to_string()], + "held_list should route exactly once through the authority site router", + ); +} + #[tokio::test] async fn dynamic_caps_mcp_discovers_compact_surface() { let harness = DynamicCapsMcpHarness::start().await; @@ -4725,6 +4841,69 @@ async fn dynamic_caps_resolve_origin_tolerates_missing_static_runtime_state_file ); } +#[tokio::test] +async fn dynamic_caps_resolve_origin_prefers_published_router_mesh_addr() { + let harness = DynamicCapsMcpHarness::start().await; + + write_json( + &harness + ._dir + .path() + .join("state") + .join("direct_local") + .join("manager-state.json"), + &json!({ + "schema": TEST_SITE_STATE_SCHEMA, + "version": TEST_SITE_STATE_VERSION, + "run_id": "test-run", + "site_id": "direct_local", + "kind": "direct", + "status": "running", + "artifact_dir": harness._dir.path().join("artifact/direct_local").display().to_string(), + "supervisor_pid": 1u32, + "router_control": test_router_control_addr(), + "router_mesh_addr": "127.0.0.1:1", + "router_identity_id": "/site/direct_local/router", + "router_public_key_b64": base64::engine::general_purpose::STANDARD.encode([7u8; 32]), + "site_controller_url": harness.base_url.clone(), + }), + ) + .expect("stale manager state should write"); + + let held: amber_mesh::dynamic_caps::HeldListResponse = harness + .post_json( + "/v1/controller/dynamic-caps/held", + &dynamic_caps::ControlDynamicHeldListRequest { + holder_component_id: "components./alice".to_string(), + }, + ) + .await; + let root_selector = held + .held + .iter() + .find(|entry| entry.entry_kind == HeldEntryKind::RootAuthority) + .and_then(|entry| entry.root_authority_selector.clone()) + .expect("alice should have a root authority selector"); + + let resolve_origin: dynamic_caps::ControlDynamicResolveOriginResponse = harness + .post_json( + "/v1/controller/dynamic-caps/resolve-origin", + &dynamic_caps::ControlDynamicResolveOriginRequest { + holder_component_id: "components./alice".to_string(), + source: dynamic_caps::DynamicCapabilityControlSourceRequest::RootAuthority { + root_authority_selector: root_selector, + }, + }, + ) + .await; + + assert_eq!( + resolve_origin.origin_peer_addr, "127.0.0.1:24000", + "resolve-origin should use the controller plan's published router mesh address instead of \ + stale manager-state data", + ); +} + #[tokio::test] async fn create_snapshot_and_destroy_exact_child() { let dir = TempDir::new().expect("temp dir"); @@ -8957,6 +9136,8 @@ volumes: &BTreeMap::new(), Some("unix:///amber/control/router-control.sock"), Some("127.0.0.1:24000"), + Some("host.docker.internal:24000"), + Some("192.168.65.254:24000"), &site_state_root.join("site-controller-state.json"), temp.path(), &temp.path().join("state"), @@ -9005,6 +9186,33 @@ volumes: .to_string(); assert_eq!(command[0].as_str(), Some("--plan")); assert_eq!(command[1].as_str(), Some(plan_path.as_str())); + #[cfg(unix)] + let expected_user = { + #[cfg(target_os = "macos")] + { + Some("0:0".to_string()) + } + #[cfg(not(target_os = "macos"))] + { + if fs::metadata("/var/run/docker.sock").is_ok() { + Some(format!("{}:{}", unsafe { libc::geteuid() }, unsafe { + libc::getegid() + })) + } else { + Some("0:0".to_string()) + } + } + }; + #[cfg(unix)] + assert_eq!( + service + .get(serde_yaml::Value::String("user".to_string())) + .and_then(serde_yaml::Value::as_str), + expected_user.as_deref(), + "compose site controller should use the host uid/gid on Linux so bind-mounted runtime \ + state stays deletable and fall back to root on macOS where Docker Desktop mediates the \ + socket mount" + ); let volumes = service .get(serde_yaml::Value::String("volumes".to_string())) .and_then(serde_yaml::Value::as_sequence) @@ -9033,8 +9241,31 @@ volumes: assert!( volumes .iter() - .any(|value| value.as_str() == Some("/var/run/docker.sock:/var/run/docker.sock")) - ); + .any(|value| { value.as_str() == Some("/var/run/docker.sock:/var/run/docker.sock") }) + ); + #[cfg(all(unix, not(target_os = "macos")))] + if let Ok(metadata) = fs::metadata("/var/run/docker.sock") { + let expected_gid = metadata.gid().to_string(); + let group_add = service + .get(serde_yaml::Value::String("group_add".to_string())) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller service should declare supplemental groups"); + if expected_gid != unsafe { libc::getegid() }.to_string() { + assert!( + group_add + .iter() + .any(|value| value.as_str() == Some(expected_gid.as_str())), + "compose site controller should join the host docker socket group when the socket \ + gid differs from the host gid", + ); + } else { + assert!( + group_add.is_empty(), + "compose site controller should not add a redundant supplemental group when the \ + host gid already owns the docker socket", + ); + } + } assert!( extra_hosts .iter() @@ -9128,6 +9359,8 @@ spec: &BTreeMap::new(), Some("amber-router:24100"), Some("127.0.0.1:24000"), + Some("host.docker.internal:24000"), + Some("192.168.65.254:24000"), &site_state_root.join("site-controller-state.json"), temp.path(), &temp.path().join("state"), @@ -9166,6 +9399,19 @@ spec: assert!(deployment_raw.contains("/amber/site/state/site-controller-plan.json")); let deployment: serde_yaml::Value = serde_yaml::from_str(&deployment_raw).expect("deployment should parse"); + let init_container = deployment["spec"]["template"]["spec"]["initContainers"] + .as_sequence() + .and_then(|containers| containers.first()) + .and_then(serde_yaml::Value::as_mapping) + .expect("deployment should contain a seed init container"); + assert_eq!( + init_container + .get(serde_yaml::Value::String("image".to_string())) + .and_then(serde_yaml::Value::as_str), + Some("ghcr.io/rdi-foundation/amber-site-controller:test"), + "kubernetes site controller seeding should reuse the site-controller image so tests and \ + CI do not depend on an extra public init image pull", + ); let container = deployment["spec"]["template"]["spec"]["containers"] .as_sequence() .and_then(|containers| containers.first()) @@ -9289,3 +9535,102 @@ fn local_site_manager_state_uses_controller_plan_when_host_state_is_absent() { Some(authority_url.as_str()) ); } + +#[test] +fn load_site_manager_state_prefers_local_controller_view_over_stale_host_state() { + let temp = TempDir::new().expect("temp dir"); + let state_root = temp.path().join("state"); + let site_state_root = state_root.join("compose-site"); + fs::create_dir_all(&site_state_root).expect("site state root should exist"); + let state = FrameworkControlState { + schema: CONTROL_STATE_SCHEMA.to_string(), + version: CONTROL_STATE_VERSION, + run_id: "test-run".to_string(), + base_scenario: ScenarioIr { + schema: amber_scenario::SCENARIO_IR_SCHEMA.to_string(), + version: amber_scenario::SCENARIO_IR_VERSION, + root: 0, + components: Vec::new(), + bindings: Vec::new(), + exports: Vec::new(), + manifest_catalog: BTreeMap::new(), + }, + run_links: Vec::new(), + placement: FrozenPlacementState { + offered_sites: BTreeMap::from([( + "compose-site".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + }, + )]), + defaults: PlacementDefaults::default(), + standby_sites: Vec::new(), + initial_active_sites: vec!["compose-site".to_string()], + dynamic_enabled_sites: vec!["compose-site".to_string()], + control_only_sites: Vec::new(), + active_site_capabilities: BTreeMap::new(), + placement_components: BTreeMap::new(), + assignments: BTreeMap::new(), + }, + generation: 0, + next_child_id: 1, + next_tx_id: 0, + id_stride: 1, + next_component_id: 0, + capability_instances: BTreeMap::new(), + journal: Vec::new(), + dynamic_capability_signing_seed_b64: mesh_dynamic_caps::signing_seed_b64( + &mesh_dynamic_caps::signing_key_from_seed( + mesh_dynamic_caps::generate_dynamic_capability_signing_seed(), + ), + ), + next_dynamic_capability_grant_id: 0, + dynamic_capability_grants: BTreeMap::new(), + dynamic_capability_grant_authority_sites: BTreeMap::new(), + dynamic_capability_journal: Vec::new(), + live_children: Vec::new(), + pending_creates: Vec::new(), + pending_destroys: Vec::new(), + }; + let state_path = site_state_root.join("site-controller-state.json"); + write_json(&state_path, &state).expect("state should write"); + let mut app = test_control_state_app(&temp, state, state_path); + fs::write( + site_state_path(&state_root, "compose-site"), + serde_json::to_vec(&serde_json::json!({ + "status": "stopped", + "kind": "compose", + "artifact_dir": "/tmp/stale-artifact", + "supervisor_pid": 1, + "compose_project": "stale-project", + "router_control": "unix:///tmp/stale-router.sock", + "router_mesh_addr": "127.0.0.1:1", + "site_controller_url": "http://stale-controller" + })) + .expect("stale manager state should serialize"), + ) + .expect("stale manager state should write"); + let controller_plan = Arc::make_mut(&mut app.controller_plan); + controller_plan.kind = SiteKind::Compose; + controller_plan.compose_project = Some("amber_test_compose".to_string()); + controller_plan.local_router_control = + Some("unix:///amber/control/router-control.sock".to_string()); + controller_plan.published_router_mesh_addr = Some("127.0.0.1:24000".to_string()); + controller_plan.authority_url = + format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{SITE_CONTROLLER_PORT}"); + + let state = load_site_manager_state(&app, "compose-site") + .expect("local controller should not trust stale host manager state"); + assert_eq!(state.status, "running"); + assert_eq!( + state.router_control.as_deref(), + Some("unix:///amber/control/router-control.sock") + ); + assert_eq!(state.router_mesh_addr.as_deref(), Some("127.0.0.1:24000")); + let authority_url = format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{SITE_CONTROLLER_PORT}"); + assert_eq!( + state.site_controller_url.as_deref(), + Some(authority_url.as_str()) + ); +} From 47a5474eeef7fa45b10851a6e260256444147972 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Thu, 16 Apr 2026 23:04:59 +0800 Subject: [PATCH 07/25] fix tests --- cli/src/direct_runtime/process.rs | 62 +---- cli/src/mixed_run.rs | 194 ++++++++++++++ cli/src/mixed_run/outside_proxy.rs | 24 +- cli/src/mixed_run/supervisor.rs | 148 +++++++++-- cli/src/mixed_run/tests.rs | 83 ++++++ cli/src/tests.rs | 65 ++++- cli/tests/linux_vm.rs | 1 + cli/tests/mixed_run/support.rs | 251 +++++++++++++++--- docker/amber-provisioner/Dockerfile | 11 +- .../src/default_runtime/compose_controller.rs | 52 +--- runtime/site-controller/src/tests.rs | 53 +--- 11 files changed, 721 insertions(+), 223 deletions(-) diff --git a/cli/src/direct_runtime/process.rs b/cli/src/direct_runtime/process.rs index 1671930d..c2324605 100644 --- a/cli/src/direct_runtime/process.rs +++ b/cli/src/direct_runtime/process.rs @@ -654,45 +654,17 @@ pub(crate) async fn terminate_children(children: &mut [ManagedChild]) { #[cfg(target_os = "linux")] pub(crate) async fn terminate_children(children: &mut [ManagedChild]) { - for child in children.iter_mut() { - if linux_pid_is_alive(child.managed_pid) { - let _ = send_sigterm(child.managed_pid); - } - if child.wrapper_pid != child.managed_pid { - let _ = send_sigterm(child.wrapper_pid); - } - } - - let deadline = Instant::now() + DIRECT_SHUTDOWN_GRACE_PERIOD; - loop { - let mut all_exited = true; - for child in children.iter_mut() { - if let Some(wrapper) = child.wrapper.as_mut() - && wrapper.try_wait().ok().flatten().is_some() - { - child.wrapper = None; - } - if linux_pid_is_alive(child.managed_pid) { - all_exited = false; - } - if child.wrapper.is_some() { - all_exited = false; - } - } - if all_exited || Instant::now() >= deadline { - break; - } - sleep(DIRECT_CHILD_POLL_INTERVAL).await; - } + let root_pids = children + .iter() + .flat_map(|child| { + let managed = (child.managed_pid != child.wrapper_pid).then_some(child.managed_pid); + [Some(child.wrapper_pid), managed] + }) + .flatten() + .collect::>(); + let _ = crate::unix_process::terminate_process_roots(&root_pids, DIRECT_SHUTDOWN_GRACE_PERIOD) + .await; - for child in children.iter_mut() { - if linux_pid_is_alive(child.managed_pid) { - let _ = kill_pid_force(child.managed_pid); - } - if child.wrapper_pid != child.managed_pid { - let _ = kill_pid_force(child.wrapper_pid); - } - } for child in children.iter_mut() { if let Some(mut wrapper) = child.wrapper.take() { let _ = wrapper.wait().await; @@ -716,13 +688,6 @@ pub(crate) async fn terminate_children(children: &mut [ManagedChild]) { } } -#[cfg(target_os = "linux")] -pub(crate) fn send_sigterm(pid: u32) -> std::result::Result<(), ()> { - let pid = i32::try_from(pid).map_err(|_| ())?; - let rc = unsafe { libc::kill(pid, libc::SIGTERM) }; - if rc == 0 { Ok(()) } else { Err(()) } -} - #[cfg(target_os = "linux")] pub(crate) fn linux_pid_is_alive(pid: u32) -> bool { let Ok(pid) = i32::try_from(pid) else { @@ -735,13 +700,6 @@ pub(crate) fn linux_pid_is_alive(pid: u32) -> bool { std::io::Error::last_os_error().raw_os_error() != Some(libc::ESRCH) } -#[cfg(target_os = "linux")] -pub(crate) fn kill_pid_force(pid: u32) -> std::result::Result<(), ()> { - let pid = i32::try_from(pid).map_err(|_| ())?; - let rc = unsafe { libc::kill(pid, libc::SIGKILL) }; - if rc == 0 { Ok(()) } else { Err(()) } -} - #[cfg(all(target_os = "linux", unix))] pub(crate) fn synthetic_failure_exit_status() -> std::process::ExitStatus { use std::os::unix::process::ExitStatusExt as _; diff --git a/cli/src/mixed_run.rs b/cli/src/mixed_run.rs index 568d5f79..d480e127 100644 --- a/cli/src/mixed_run.rs +++ b/cli/src/mixed_run.rs @@ -85,8 +85,11 @@ const PROCESS_SHUTDOWN_GRACE_PERIOD: Duration = Duration::from_secs(10); const SITE_SUPERVISOR_STOP_TIMEOUT: Duration = Duration::from_secs(60); const FORCED_SUPERVISOR_EXIT_GRACE_PERIOD: Duration = Duration::from_secs(5); const KUBERNETES_NAMESPACE_READY_TIMEOUT: Duration = Duration::from_secs(60); +const KUBERNETES_NAMESPACE_DELETE_TIMEOUT: Duration = Duration::from_secs(30); +const KUBERNETES_NAMESPACE_FORCE_DELETE_GRACE_PERIOD: Duration = Duration::from_secs(10); const KUBERNETES_WORKLOAD_READY_TIMEOUT: Duration = Duration::from_secs(120); const KUBERNETES_SITE_READY_BUFFER: Duration = Duration::from_secs(30); +const KUBERNETES_NAMESPACE_DELETE_POLL_INTERVAL: Duration = Duration::from_millis(250); const VM_LOCAL_TARGET_READY_TIMEOUT: Duration = Duration::from_secs(1); const DEFAULT_EXTERNAL_ENV_FILE: &str = "router-external.env"; @@ -1170,6 +1173,197 @@ fn stop_marker_path(run_root: &Path) -> PathBuf { run_root.join("stop-requested") } +fn kubectl_command_with_bin(kubectl_bin: &Path, context: Option<&str>) -> Command { + let mut cmd = Command::new(kubectl_bin); + if let Some(context) = context { + cmd.arg("--context").arg(context); + } + cmd +} + +fn kubectl_stderr(output: &std::process::Output) -> String { + String::from_utf8_lossy(&output.stderr).trim().to_string() +} + +fn kubectl_stdout(output: &std::process::Output) -> String { + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +fn kubectl_detail(output: &std::process::Output) -> String { + let stdout = kubectl_stdout(output); + let stderr = kubectl_stderr(output); + match (stdout.is_empty(), stderr.is_empty()) { + (false, false) => format!("stdout:\n{stdout}\nstderr:\n{stderr}"), + (false, true) => format!("stdout:\n{stdout}"), + (true, false) => format!("stderr:\n{stderr}"), + (true, true) => format!("status {}", output.status), + } +} + +fn kubectl_is_not_found(output: &std::process::Output) -> bool { + let stderr = kubectl_stderr(output); + stderr.contains("(NotFound)") || stderr.contains("not found") +} + +fn kubectl_force_delete_namespace_pods_with_bin( + kubectl_bin: &Path, + context: Option<&str>, + namespace: &str, +) -> Result<()> { + let output = kubectl_command_with_bin(kubectl_bin, context) + .args([ + "-n", + namespace, + "delete", + "pods", + "--all", + "--ignore-not-found", + "--force", + "--grace-period=0", + ]) + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to force-delete remaining pods in kubernetes namespace `{namespace}`") + })?; + if !output.status.success() && !kubectl_is_not_found(&output) { + return Err(miette::miette!( + "force-delete of remaining pods in kubernetes namespace `{namespace}` failed:\n{}", + kubectl_detail(&output) + )); + } + Ok(()) +} + +fn kubectl_list_namespace_pods_with_bin( + kubectl_bin: &Path, + context: Option<&str>, + namespace: &str, +) -> Result> { + let output = kubectl_command_with_bin(kubectl_bin, context) + .args(["-n", namespace, "get", "pods", "--ignore-not-found", "-o", "name"]) + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to list remaining pods in kubernetes namespace `{namespace}`") + })?; + if !output.status.success() { + if kubectl_is_not_found(&output) { + return Ok(Vec::new()); + } + return Err(miette::miette!( + "listing remaining pods in kubernetes namespace `{namespace}` failed:\n{}", + kubectl_detail(&output) + )); + } + Ok(kubectl_stdout(&output) + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(ToOwned::to_owned) + .collect()) +} + +pub(super) fn stop_kubernetes_namespace( + context: Option<&str>, + namespace: &str, + timeout: Duration, +) -> Result<()> { + stop_kubernetes_namespace_with_kubectl(Path::new("kubectl"), context, namespace, timeout) +} + +fn stop_kubernetes_namespace_with_kubectl( + kubectl_bin: &Path, + context: Option<&str>, + namespace: &str, + timeout: Duration, +) -> Result<()> { + let delete_output = kubectl_command_with_bin(kubectl_bin, context) + .args([ + "delete", + "namespace", + namespace, + "--ignore-not-found", + "--wait=false", + ]) + .output() + .into_diagnostic() + .wrap_err_with(|| format!("failed to start deletion of kubernetes namespace `{namespace}`"))?; + if !delete_output.status.success() && !kubectl_is_not_found(&delete_output) { + return Err(miette::miette!( + "starting deletion of kubernetes namespace `{namespace}` failed:\n{}", + kubectl_detail(&delete_output) + )); + } + + let mut deadline = Instant::now() + timeout; + let mut forced_pod_cleanup = false; + let mut last_namespace_detail = None::; + loop { + let get_output = kubectl_command_with_bin(kubectl_bin, context) + .args(["get", "namespace", namespace, "-o", "json"]) + .output() + .into_diagnostic() + .wrap_err_with(|| format!("failed to query kubernetes namespace `{namespace}`"))?; + if !get_output.status.success() { + if kubectl_is_not_found(&get_output) { + return Ok(()); + } + last_namespace_detail = Some(kubectl_detail(&get_output)); + } else { + last_namespace_detail = Some(kubectl_stdout(&get_output)); + } + + if Instant::now() >= deadline { + if !forced_pod_cleanup { + kubectl_force_delete_namespace_pods_with_bin(kubectl_bin, context, namespace)?; + let retry_output = kubectl_command_with_bin(kubectl_bin, context) + .args([ + "delete", + "namespace", + namespace, + "--ignore-not-found", + "--wait=false", + ]) + .output() + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to retry deletion of kubernetes namespace `{namespace}`") + })?; + if !retry_output.status.success() && !kubectl_is_not_found(&retry_output) { + return Err(miette::miette!( + "retrying deletion of kubernetes namespace `{namespace}` failed:\n{}", + kubectl_detail(&retry_output) + )); + } + forced_pod_cleanup = true; + deadline = Instant::now() + KUBERNETES_NAMESPACE_FORCE_DELETE_GRACE_PERIOD; + std::thread::sleep(KUBERNETES_NAMESPACE_DELETE_POLL_INTERVAL); + continue; + } + + let remaining_pods = + kubectl_list_namespace_pods_with_bin(kubectl_bin, context, namespace)?; + let pod_detail = if remaining_pods.is_empty() { + "remaining pods: none".to_string() + } else { + format!("remaining pods: {}", remaining_pods.join(", ")) + }; + let namespace_detail = last_namespace_detail + .as_deref() + .filter(|detail| !detail.is_empty()) + .map(|detail| format!("last namespace response:\n{detail}")) + .unwrap_or_else(|| "last namespace response: ".to_string()); + return Err(miette::miette!( + "timed out waiting for kubernetes namespace `{namespace}` to delete after {}s\n{pod_detail}\n{namespace_detail}", + timeout.as_secs() + KUBERNETES_NAMESPACE_FORCE_DELETE_GRACE_PERIOD.as_secs() + )); + } + + std::thread::sleep(KUBERNETES_NAMESPACE_DELETE_POLL_INTERVAL); + } +} + async fn wait_for_stop_request(stop_requested: &AtomicBool, run_root: &Path) { loop { if stop_requested.load(Ordering::SeqCst) || stop_marker_path(run_root).exists() { diff --git a/cli/src/mixed_run/outside_proxy.rs b/cli/src/mixed_run/outside_proxy.rs index d25168c7..aa3ebf50 100644 --- a/cli/src/mixed_run/outside_proxy.rs +++ b/cli/src/mixed_run/outside_proxy.rs @@ -358,22 +358,14 @@ pub(super) async fn stop_site_from_receipt( send_sigterm(pid); } if let Some(namespace) = site.kubernetes_namespace.as_deref() { - let status = kubectl_command(site.context.as_deref()) - .arg("delete") - .arg("namespace") - .arg(namespace) - .arg("--ignore-not-found") - .status() - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to stop orphaned kubernetes site `{namespace}`") - })?; - if !status.success() { - return Err(miette::miette!( - "orphaned kubernetes site `{namespace}` failed to stop with status \ - {status}" - )); - } + stop_kubernetes_namespace( + site.context.as_deref(), + namespace, + KUBERNETES_NAMESPACE_DELETE_TIMEOUT, + ) + .wrap_err_with(|| { + format!("failed to stop orphaned kubernetes site `{namespace}`") + })?; } } } diff --git a/cli/src/mixed_run/supervisor.rs b/cli/src/mixed_run/supervisor.rs index c842c4c1..1a35b3a7 100644 --- a/cli/src/mixed_run/supervisor.rs +++ b/cli/src/mixed_run/supervisor.rs @@ -346,8 +346,10 @@ pub(super) async fn wait_for_site_ready( )); } if Instant::now() >= deadline { + let diagnostics = + site_startup_timeout_diagnostics(site_plan, site_state_root, &state_path); return Err(miette::miette!( - "timed out waiting for site `{site_id}` to become ready" + "timed out waiting for site `{site_id}` to become ready{diagnostics}" )); } sleep(Duration::from_millis(200)).await; @@ -1016,6 +1018,125 @@ pub(super) fn parse_container_runtime_status(raw: &str) -> Option<(&str, Option< Some((status, health)) } +fn tail_lines(raw: &str, keep: usize) -> String { + let lines = raw.lines().collect::>(); + let start = lines.len().saturating_sub(keep); + lines[start..].join("\n") +} + +fn timeout_file_section(label: &str, path: &Path, keep_lines: usize) -> Option { + let raw = fs::read_to_string(path).ok()?; + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + Some(format!( + "{label} ({}):\n{}", + path.display(), + tail_lines(trimmed, keep_lines) + )) +} + +fn timeout_command_section(label: &str, command: &mut Command) -> Option { + let output = command.output().ok()?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let body = if output.status.success() { + stdout.trim().to_string() + } else if stdout.trim().is_empty() { + stderr.trim().to_string() + } else if stderr.trim().is_empty() { + stdout.trim().to_string() + } else { + format!("stdout:\n{}\n\nstderr:\n{}", stdout.trim(), stderr.trim()) + }; + if body.is_empty() { + return None; + } + Some(format!("{label}:\n{body}")) +} + +fn compose_site_timeout_sections(plan: &SiteSupervisorPlan) -> Vec { + let mut sections = Vec::new(); + if let Ok(status) = inspect_compose_site_controller_status(plan) { + let summary = status + .map(|(status, health)| match health { + Some(health) => format!("{status} {health}"), + None => status, + }) + .unwrap_or_else(|| "unavailable".to_string()); + sections.push(format!( + "compose site controller status (`{}`): {summary}", + compose_site_controller_container_name(plan) + .unwrap_or_else(|| "".to_string()) + )); + } + if let Some(compose_ps) = timeout_command_section( + "docker compose ps --all", + compose_command(plan.compose_project.as_deref(), Path::new(&plan.artifact_dir)) + .arg("ps") + .arg("--all"), + ) { + sections.push(compose_ps); + } + if let Some(container_name) = compose_site_controller_container_name(plan) { + if let Some(inspect_state) = timeout_command_section( + &format!("docker inspect state for `{container_name}`"), + Command::new("docker") + .arg("inspect") + .arg("--format") + .arg("{{json .State}}") + .arg(&container_name), + ) { + sections.push(inspect_state); + } + if let Some(container_logs) = timeout_command_section( + &format!("docker logs `{container_name}`"), + Command::new("docker") + .arg("logs") + .arg("--tail") + .arg("200") + .arg(&container_name), + ) { + sections.push(container_logs); + } + } + sections +} + +fn site_startup_timeout_diagnostics( + site_plan: &RunSitePlan, + site_state_root: &Path, + state_path: &Path, +) -> String { + let mut sections = Vec::new(); + if let Some(section) = timeout_file_section("manager state", state_path, 200) { + sections.push(section); + } + if let Some(section) = + timeout_file_section("supervisor log", &site_state_root.join("supervisor.log"), 200) + { + sections.push(section); + } + if let Some(section) = timeout_file_section("site log", &site_state_root.join("site.log"), 200) + { + sections.push(section); + } + if site_plan.site.kind == SiteKind::Compose + && let Ok(plan) = read_json::( + &site_supervisor_plan_path(site_state_root), + "site supervisor plan", + ) + { + sections.extend(compose_site_timeout_sections(&plan)); + } + if sections.is_empty() { + String::new() + } else { + format!("\nstartup diagnostics:\n{}", sections.join("\n\n")) + } +} + fn inspect_compose_site_controller_status( plan: &SiteSupervisorPlan, ) -> Result)>> { @@ -1238,22 +1359,17 @@ pub(super) async fn cleanup_site( } SiteKind::Kubernetes => { if let Some(namespace) = plan.kubernetes_namespace.as_deref() { - let status = kubectl_command(plan.context.as_deref()) - .arg("delete") - .arg("namespace") - .arg(namespace) - .arg("--ignore-not-found") - .status() - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to stop kubernetes site `{}`", plan.site_id) - })?; - if !status.success() { - return Err(miette::miette!( - "kubernetes site `{}` failed to stop with status {status}", + stop_kubernetes_namespace( + plan.context.as_deref(), + namespace, + KUBERNETES_NAMESPACE_DELETE_TIMEOUT, + ) + .wrap_err_with(|| { + format!( + "failed to stop kubernetes site `{}` in namespace `{namespace}`", plan.site_id - )); - } + ) + })?; } } SiteKind::Direct | SiteKind::Vm => {} diff --git a/cli/src/mixed_run/tests.rs b/cli/src/mixed_run/tests.rs index a9b8d3de..fb7d5ac0 100644 --- a/cli/src/mixed_run/tests.rs +++ b/cli/src/mixed_run/tests.rs @@ -9,6 +9,9 @@ use tempfile::TempDir; use super::*; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt as _; + #[test] fn site_state_paths_are_site_scoped() { let root = Path::new("/tmp/amber-run/state"); @@ -170,6 +173,26 @@ fn site_controller_image_includes_the_amber_cli_binary() { ); } +#[test] +fn provisioner_image_supports_debug_builds() { + let dockerfile = fs::read_to_string( + Path::new(env!("CARGO_MANIFEST_DIR")).join("../docker/amber-provisioner/Dockerfile"), + ) + .expect("provisioner Dockerfile should read"); + assert!( + dockerfile.contains("ARG BUILD_MODE=release"), + "provisioner image should accept the shared BUILD_MODE argument so mixed-run tests can use \ + debug builds:\n{dockerfile}" + ); + assert!( + dockerfile.contains("if [ \"$BUILD_MODE\" = \"release\" ]; then") + && dockerfile.contains("cargo build -p amber-provisioner --release --locked") + && dockerfile.contains("cargo build -p amber-provisioner --locked") + && dockerfile.contains("\"${build_dir}\"/amber-provisioner"), + "provisioner image should support both release and debug output paths:\n{dockerfile}" + ); +} + #[test] fn site_controller_command_prefers_fresh_amber_binary_under_cargo_tests() { let temp = TempDir::new().expect("temp dir"); @@ -375,3 +398,63 @@ fn reserve_loopback_port_shares_allocator_with_site_controller_runtime() { ); } } + +#[cfg(unix)] +#[test] +fn stop_kubernetes_namespace_force_deletes_stuck_pods_before_retrying() { + let temp = TempDir::new().expect("temp dir"); + let kubectl = temp.path().join("kubectl"); + let log_path = temp.path().join("kubectl.log"); + let state_path = temp.path().join("namespace-state"); + fs::write( + &kubectl, + format!( + "#!/bin/sh\nset -eu\nlog_path='{}'\nstate_path='{}'\nprintf '%s\\n' \"$*\" >> \ + \"$log_path\"\nif [ \"${{1:-}}\" = \"--context\" ]; then\n shift 2\nfi\nstate=alive\n\ + if [ -f \"$state_path\" ]; then\n state=$(cat \"$state_path\")\nfi\nif [ \"${{1:-}}\" \ + = \"delete\" ] && [ \"${{2:-}}\" = \"namespace\" ]; then\n exit 0\nfi\nif [ \ + \"${{1:-}}\" = \"get\" ] && [ \"${{2:-}}\" = \"namespace\" ]; then\n if [ \"$state\" = \ + \"gone\" ]; then\n echo 'Error from server (NotFound): namespaces \"'\"${{3:-}}\"'\" \ + not found' >&2\n exit 1\n fi\n printf '{{\"metadata\":{{\"name\":\"%s\",\ + \"deletionTimestamp\":\"2026-04-16T02:33:55Z\"}}}}\\n' \"${{3:-}}\"\n exit 0\nfi\nif \ + [ \"${{1:-}}\" = \"-n\" ] && [ \"${{2:-}}\" = \"test-ns\" ] && [ \"${{3:-}}\" = \ + \"delete\" ] && [ \"${{4:-}}\" = \"pods\" ]; then\n printf 'gone' > \"$state_path\"\n \ + exit 0\nfi\necho \"unexpected kubectl invocation: $*\" >&2\nexit 1\n", + log_path.display(), + state_path.display(), + ), + ) + .expect("kubectl stub"); + let mut permissions = fs::metadata(&kubectl) + .expect("kubectl metadata") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&kubectl, permissions).expect("kubectl chmod"); + + stop_kubernetes_namespace_with_kubectl( + &kubectl, + Some("test-context"), + "test-ns", + Duration::from_millis(20), + ) + .expect("namespace stop should force-delete remaining pods and retry"); + + let log = fs::read_to_string(&log_path).expect("kubectl log"); + let delete_namespace_calls = log + .lines() + .filter(|line| { + line.contains("delete namespace test-ns --ignore-not-found --wait=false") + }) + .count(); + assert_eq!( + delete_namespace_calls, 2, + "namespace deletion should be retried after forced pod cleanup:\n{log}" + ); + assert!( + log.lines().any(|line| { + line.contains("-n test-ns delete pods --all --ignore-not-found --force \ + --grace-period=0") + }), + "forced pod cleanup should run before the retry:\n{log}" + ); +} diff --git a/cli/src/tests.rs b/cli/src/tests.rs index 771c1849..c1b23033 100644 --- a/cli/src/tests.rs +++ b/cli/src/tests.rs @@ -903,12 +903,23 @@ async fn cleanup_direct_runtime_terminates_wrapper_descendants() { fs::write(&runtime_state_path, "{}").expect("state file should be written"); let child_pid_path = runtime_root.join("child.pid"); - let child = TokioCommand::new("sh") + // Use a real wrapper process so the test records the long-lived descendant PID rather than + // relying on shell job-control or `setsid` utility behavior. + let child = TokioCommand::new("python3") .arg("-c") - .arg(format!( - "setsid sleep 30 >/dev/null 2>&1 & echo $! > {} ; wait", - child_pid_path.display() - )) + .arg( + r#" +import pathlib +import subprocess +import sys + +pid_path = pathlib.Path(sys.argv[1]) +child = subprocess.Popen(["sleep", "30"], start_new_session=True) +pid_path.write_text(f"{child.pid}\n", encoding="utf-8") +child.wait() +"#, + ) + .arg(&child_pid_path) .spawn() .expect("wrapper should spawn"); let mut children = vec![ManagedChild { @@ -939,6 +950,20 @@ async fn cleanup_direct_runtime_terminates_wrapper_descendants() { .trim() .parse::() .expect("descendant pid should parse"); + let wrapper_pid = children[0] + .wrapper + .as_ref() + .and_then(tokio::process::Child::id) + .expect("wrapper pid should be available"); + let tree_deadline = Instant::now() + Duration::from_secs(5); + while descendant_parent_pid(child_pid) != Some(wrapper_pid) && Instant::now() < tree_deadline { + sleep(Duration::from_millis(25)).await; + } + assert_eq!( + descendant_parent_pid(child_pid), + Some(wrapper_pid), + "descendant should appear in the process table under its wrapper before cleanup" + ); cleanup_direct_runtime( &mut children, @@ -950,15 +975,43 @@ async fn cleanup_direct_runtime_terminates_wrapper_descendants() { .await; let stopped = !crate::unix_process::pid_is_alive(child_pid); + let survivor = if stopped { + None + } else { + Some( + std::process::Command::new("ps") + .args(["-o", "pid=,ppid=,pgid=,stat=,command=", "-p"]) + .arg(child_pid.to_string()) + .output() + .ok() + .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_string()) + .filter(|line| !line.is_empty()) + .unwrap_or_else(|| "".to_string()), + ) + }; if !stopped { let _ = unsafe { libc::kill(child_pid as i32, libc::SIGKILL) }; } assert!( stopped, - "cleanup should terminate descendants that outlive their wrapper" + "cleanup should terminate descendants that outlive their wrapper; surviving process: {}", + survivor.unwrap_or_default() ); } +#[cfg(unix)] +fn descendant_parent_pid(pid: u32) -> Option { + let output = std::process::Command::new("ps") + .args(["-o", "ppid=", "-p"]) + .arg(pid.to_string()) + .output() + .ok()?; + if !output.status.success() { + return None; + } + String::from_utf8_lossy(&output.stdout).trim().parse().ok() +} + #[test] fn write_direct_runtime_state_preserves_projected_router_mesh_port() { let plan_root = tempfile::tempdir().expect("temp dir should be created"); diff --git a/cli/tests/linux_vm.rs b/cli/tests/linux_vm.rs index b73f5f53..9e43932e 100644 --- a/cli/tests/linux_vm.rs +++ b/cli/tests/linux_vm.rs @@ -796,6 +796,7 @@ impl LinuxVmHarness { &format!( "set -euxo pipefail\n. \"$HOME/.cargo/env\"\ncd {guest}\nexport \ AMBER_TEST_KEEP_OUTPUTS=1\nexport \ + AMBER_TEST_INTERNAL_IMAGE_BUILD_MODE=debug\nexport \ AMBER_VM_SMOKE_BASE_IMAGE=\"$HOME/{image_filename}\"\nexport \ AMBER_MIXED_RUN_BASE_IMAGE=\"$HOME/{image_filename}\"\nexport \ CARGO_TARGET_DIR=\"$HOME/amber-target\"\nexport CARGO_TERM_QUIET=true\nexport \ diff --git a/cli/tests/mixed_run/support.rs b/cli/tests/mixed_run/support.rs index 3697b5a2..568421db 100644 --- a/cli/tests/mixed_run/support.rs +++ b/cli/tests/mixed_run/support.rs @@ -587,14 +587,93 @@ pub(crate) fn append_debug_file(out: &mut String, label: &str, path: &Path) { )); } +fn manager_state_value(state_path: &Path) -> Option { + fs::read_to_string(state_path) + .ok() + .and_then(|raw| serde_json::from_str::(&raw).ok()) +} + +fn append_docker_project_debug(out: &mut String, project: &str) { + let project_filter = format!("label=com.docker.compose.project={project}"); + let ps_output = match Command::new("docker") + .args([ + "ps", + "-a", + "--filter", + project_filter.as_str(), + "--format", + "{{.ID}}\t{{.Names}}\t{{.Status}}", + ]) + .output() + { + Ok(output) => output, + Err(err) => { + out.push_str(&format!( + "\ndocker compose project `{project}` status query failed:\n{err}\n" + )); + return; + } + }; + + if !ps_output.status.success() { + out.push_str(&format!( + "\ndocker compose project `{project}` status query failed\nstdout:\n{}\nstderr:\n{}\n", + String::from_utf8_lossy(&ps_output.stdout), + String::from_utf8_lossy(&ps_output.stderr), + )); + return; + } + + let containers = String::from_utf8_lossy(&ps_output.stdout) + .lines() + .filter_map(|line| { + let mut parts = line.splitn(3, '\t'); + Some(( + parts.next()?.trim().to_string(), + parts.next()?.trim().to_string(), + parts.next()?.trim().to_string(), + )) + }) + .collect::>(); + + if containers.is_empty() { + out.push_str(&format!( + "\ndocker compose project `{project}` containers:\n\n" + )); + return; + } + + out.push_str(&format!("\ndocker compose project `{project}` containers:\n")); + for (_, name, status) in &containers { + out.push_str(&format!("{name}\t{status}\n")); + } + + for (id, name, status) in containers { + let logs_output = match Command::new("docker") + .args(["logs", "--tail", "80", id.as_str()]) + .output() + { + Ok(output) => output, + Err(err) => { + out.push_str(&format!( + "\ndocker logs {name} ({status}) failed:\n{err}\n" + )); + continue; + } + }; + out.push_str(&format!( + "\ndocker logs {name} ({status}):\nstdout:\n{}\nstderr:\n{}\n", + String::from_utf8_lossy(&logs_output.stdout), + String::from_utf8_lossy(&logs_output.stderr), + )); + } +} + pub(crate) fn site_debug_context(run_root: &Path, site_id: &str) -> String { let state_root = run_root.join("state").join(site_id); + let manager_state_path = state_root.join("manager-state.json"); let mut out = String::new(); - append_debug_file( - &mut out, - "manager state", - &state_root.join("manager-state.json"), - ); + append_debug_file(&mut out, "manager state", &manager_state_path); append_debug_file( &mut out, "supervisor log", @@ -606,6 +685,11 @@ pub(crate) fn site_debug_context(run_root: &Path, site_id: &str) -> String { &state_root.join("port-forward.log"), ); append_debug_file(&mut out, "site log", &state_root.join("site.log")); + if let Some(project) = manager_state_value(&manager_state_path) + .and_then(|state| state["compose_project"].as_str().map(ToOwned::to_owned)) + { + append_docker_project_debug(&mut out, &project); + } out } @@ -723,6 +807,22 @@ pub(crate) fn use_prebuilt_images() -> bool { env::var_os("AMBER_TEST_USE_PREBUILT_IMAGES").is_some() } +fn internal_image_build_mode() -> &'static str { + static MODE: OnceLock<&'static str> = OnceLock::new(); + MODE.get_or_init(|| match env::var("AMBER_TEST_INTERNAL_IMAGE_BUILD_MODE") { + Ok(mode) if mode == "release" => "release", + Ok(mode) if mode == "debug" => "debug", + Ok(mode) => panic!( + "AMBER_TEST_INTERNAL_IMAGE_BUILD_MODE must be `debug` or `release`, got `{mode}`" + ), + Err(env::VarError::NotPresent) => "debug", + Err(env::VarError::NotUnicode(value)) => panic!( + "AMBER_TEST_INTERNAL_IMAGE_BUILD_MODE must be valid UTF-8, got {:?}", + value + ), + }) +} + pub(crate) fn image_platform_opt(tag: &str) -> Option { let output = Command::new("docker") .arg("image") @@ -761,12 +861,19 @@ pub(crate) fn ensure_docker_image(tag: &str, dockerfile: &Path) { for attempt in 1..=3 { let mut command = Command::new("docker"); if docker_supports_buildx() { - command.arg("buildx").arg("build").arg("--load"); + command + .arg("buildx") + .arg("build") + .arg("--load") + .arg("--progress") + .arg("plain"); } else { command.env("DOCKER_BUILDKIT", "1"); command.arg("build"); } let status = command + .arg("--build-arg") + .arg(format!("BUILD_MODE={}", internal_image_build_mode())) .arg("-t") .arg(tag) .arg("-f") @@ -953,7 +1060,12 @@ fn ensure_local_dev_image_tag_overrides() { } let mut hasher = DefaultHasher::new(); workspace_root().hash(&mut hasher); - let tag = format!("dev-mixed-run-{:016x}", hasher.finish()); + internal_image_build_mode().hash(&mut hasher); + let tag = format!( + "dev-mixed-run-{}-{:016x}", + internal_image_build_mode(), + hasher.finish() + ); let overrides = INTERNAL_IMAGE_OVERRIDE_KEYS .iter() .map(|key| format!("{key}={tag}")) @@ -1015,27 +1127,47 @@ fn kind_load_docker_image(cluster_name: &str, image: &str) -> Result<(), String> } fn kind_load_image_archive(cluster_name: &str, image: &str) -> Result<(), String> { - let mut docker = Command::new("docker") + let archive = tempfile::Builder::new() + .prefix("amber-kind-image-") + .suffix(".tar") + .tempfile() + .map_err(|err| { + format!("failed to create temporary image archive for {image} in {cluster_name}: {err}") + })?; + kind_load_image_archive_with_binaries(Path::new("kind"), Path::new("docker"), cluster_name, image, archive) +} + +fn kind_load_image_archive_with_binaries( + kind_bin: &Path, + docker_bin: &Path, + cluster_name: &str, + image: &str, + archive: tempfile::NamedTempFile, +) -> Result<(), String> { + let archive_path = archive.path().to_path_buf(); + let docker_output = Command::new(docker_bin) .arg("image") .arg("save") + .arg("--output") + .arg(&archive_path) .arg(image) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() + .output() .map_err(|err| { format!("failed to run `docker image save` for {image} in {cluster_name}: {err}") })?; - let docker_stdout = docker - .stdout - .take() - .ok_or_else(|| format!("docker image save for {image} did not expose stdout"))?; - let kind = Command::new("kind") + if !docker_output.status.success() { + return Err(format_command_output( + &format!("docker image save failed for {image}"), + &docker_output, + )); + } + + let kind = Command::new(kind_bin) .arg("load") .arg("image-archive") .arg("--name") .arg(cluster_name) - .arg("-") - .stdin(Stdio::from(docker_stdout)) + .arg(&archive_path) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .output() @@ -1045,30 +1177,14 @@ fn kind_load_image_archive(cluster_name: &str, image: &str) -> Result<(), String {err}" ) })?; - let docker_output = docker.wait_with_output().map_err(|err| { - format!("failed to collect `docker image save` output for {image}: {err}") - })?; - if kind.status.success() && docker_output.status.success() { + if kind.status.success() { return Ok(()); } - let mut message = String::new(); - if !docker_output.status.success() { - message.push_str(&format_command_output( - &format!("docker image save failed for {image}"), - &docker_output, - )); - } - if !kind.status.success() { - if !message.is_empty() { - message.push_str("\n\n"); - } - message.push_str(&format_command_output( - &format!("kind load image-archive failed for {image} in cluster {cluster_name}"), - &kind, - )); - } - Err(message) + Err(format_command_output( + &format!("kind load image-archive failed for {image} in cluster {cluster_name}"), + &kind, + )) } fn format_command_output(label: &str, output: &std::process::Output) -> String { @@ -2809,6 +2925,9 @@ pub(crate) fn namespace_exists(namespace: &str, kubeconfig: &Path, context: &str #[cfg(test)] mod tests { + #[cfg(unix)] + use std::os::unix::fs::PermissionsExt as _; + use super::*; #[test] @@ -2905,4 +3024,58 @@ mod tests { state_path.display() ); } + + #[cfg(unix)] + #[test] + fn kind_load_image_archive_uses_real_archive_path() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let docker = temp.path().join("docker"); + let kind = temp.path().join("kind"); + let log_path = temp.path().join("commands.log"); + + fs::write( + &docker, + format!( + "#!/bin/sh\nset -eu\nprintf 'docker %s\\n' \"$*\" >> '{}'\nif [ \"$1\" = image ] && [ \"$2\" = save ] && [ \"$3\" = --output ]; then\n printf 'archive' > \"$4\"\n exit 0\nfi\necho \"unexpected docker invocation: $*\" >&2\nexit 1\n", + log_path.display() + ), + ) + .expect("docker stub should write"); + fs::write( + &kind, + format!( + "#!/bin/sh\nset -eu\nprintf 'kind %s\\n' \"$*\" >> '{}'\narchive=\"${{5:-}}\"\nif [ \"$1\" = load ] && [ \"$2\" = image-archive ] && [ \"$3\" = --name ] && [ \"$5\" != '-' ] && [ -f \"$archive\" ]; then\n exit 0\nfi\necho \"unexpected kind invocation: $*\" >&2\nexit 1\n", + log_path.display() + ), + ) + .expect("kind stub should write"); + for path in [&docker, &kind] { + let mut permissions = fs::metadata(path) + .expect("stub metadata should read") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions).expect("stub should chmod"); + } + + let archive = tempfile::Builder::new() + .prefix("kind-archive-") + .suffix(".tar") + .tempfile_in(temp.path()) + .expect("temp archive should create"); + kind_load_image_archive_with_binaries(&kind, &docker, "test-cluster", "test:image", archive) + .expect("kind image-archive fallback should use a real archive path"); + + let log = fs::read_to_string(&log_path).expect("commands log should read"); + assert!( + log.lines().any(|line| line.contains("docker image save --output ")), + "docker should save to a concrete archive path:\n{log}" + ); + assert!( + log.lines().any(|line| { + line.starts_with("kind load image-archive --name test-cluster ") + && !line.ends_with(" -") + }), + "kind should receive the archive filename, not stdin:\n{log}" + ); + } } diff --git a/docker/amber-provisioner/Dockerfile b/docker/amber-provisioner/Dockerfile index 61e9c35f..9128b491 100644 --- a/docker/amber-provisioner/Dockerfile +++ b/docker/amber-provisioner/Dockerfile @@ -51,9 +51,16 @@ COPY compiler ./compiler COPY runtime/mesh ./runtime/mesh COPY runtime/provisioner ./runtime/provisioner +ARG BUILD_MODE=release RUN target=$(cat /tmp/rust-target) && \ - cargo build -p amber-provisioner --release --locked --target "${target}" && \ - install -D -m 0755 /app/target/"${target}"/release/amber-provisioner /out/amber-provisioner && \ + if [ "$BUILD_MODE" = "release" ]; then \ + cargo build -p amber-provisioner --release --locked --target "${target}"; \ + build_dir=release; \ + else \ + cargo build -p amber-provisioner --locked --target "${target}"; \ + build_dir=debug; \ + fi && \ + install -D -m 0755 /app/target/"${target}"/"${build_dir}"/amber-provisioner /out/amber-provisioner && \ mkdir -p /out/amber/plan /out/amber/provision && \ touch /out/amber/plan/mesh-provision-plan.json diff --git a/runtime/site-controller/src/default_runtime/compose_controller.rs b/runtime/site-controller/src/default_runtime/compose_controller.rs index 1a8b11b6..343460ad 100644 --- a/runtime/site-controller/src/default_runtime/compose_controller.rs +++ b/runtime/site-controller/src/default_runtime/compose_controller.rs @@ -1,5 +1,3 @@ -#[cfg(all(unix, not(target_os = "macos")))] -use std::os::unix::fs::MetadataExt; use std::{fs, path::Path}; use serde_json::json; @@ -14,33 +12,6 @@ const COMPOSE_ROUTER_CONTROL_SOCKET_DIR: &str = "/amber/control"; const COMPOSE_ROUTER_CONTROL_VOLUME_NAME: &str = "amber-router-control"; const DOCKER_SOCK_PATH: &str = "/var/run/docker.sock"; -#[cfg(target_os = "macos")] -fn compose_site_controller_user() -> String { - // Docker Desktop's socket mount on macOS is mediated by the host, so the controller needs - // root inside the Compose site to reach it reliably. - "0:0".to_string() -} - -#[cfg(all(unix, not(target_os = "macos")))] -fn compose_site_controller_user() -> String { - if fs::metadata(DOCKER_SOCK_PATH).is_ok() { - return format!("{}:{}", unsafe { libc::geteuid() }, unsafe { - libc::getegid() - }); - } - "0:0".to_string() -} - -#[cfg(not(unix))] -fn compose_site_controller_user() -> String { - "0:0".to_string() -} - -#[cfg(target_os = "macos")] -fn compose_site_controller_group_add() -> Vec { - Vec::new() -} - fn compose_site_controller_env(plan: &SiteControllerPlan) -> serde_json::Value { serde_json::Value::Object( plan.launch_env @@ -50,22 +21,6 @@ fn compose_site_controller_env(plan: &SiteControllerPlan) -> serde_json::Value { ) } -#[cfg(all(unix, not(target_os = "macos")))] -fn compose_site_controller_group_add() -> Vec { - if let Ok(metadata) = fs::metadata(DOCKER_SOCK_PATH) { - let socket_gid = metadata.gid(); - if socket_gid != unsafe { libc::getegid() } { - return vec![socket_gid.to_string()]; - } - } - Vec::new() -} - -#[cfg(not(unix))] -fn compose_site_controller_group_add() -> Vec { - Vec::new() -} - pub fn inject_compose_site_controller( artifact_root: &Path, plan: &SiteControllerPlan, @@ -103,11 +58,12 @@ pub fn inject_compose_site_controller( COMPOSE_MESH_NETWORK_NAME.to_string(), serde_json::Value::Object(serde_json::Map::new()), )]); - let group_add = compose_site_controller_group_add(); let service = json!({ "image": controller_image, - "user": compose_site_controller_user(), - "group_add": group_add, + // The controller must be able to reach both the router-control volume and the mounted + // run root. The router-control init service locks `/amber/control` down to `0700`, so a + // host-derived UID/GID breaks Linux compose startup. + "user": "0:0", "command": ["--plan", plan_path.display().to_string()], "environment": compose_site_controller_env(plan), "networks": networks, diff --git a/runtime/site-controller/src/tests.rs b/runtime/site-controller/src/tests.rs index f4583a65..6191e79b 100644 --- a/runtime/site-controller/src/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -9186,32 +9186,13 @@ volumes: .to_string(); assert_eq!(command[0].as_str(), Some("--plan")); assert_eq!(command[1].as_str(), Some(plan_path.as_str())); - #[cfg(unix)] - let expected_user = { - #[cfg(target_os = "macos")] - { - Some("0:0".to_string()) - } - #[cfg(not(target_os = "macos"))] - { - if fs::metadata("/var/run/docker.sock").is_ok() { - Some(format!("{}:{}", unsafe { libc::geteuid() }, unsafe { - libc::getegid() - })) - } else { - Some("0:0".to_string()) - } - } - }; - #[cfg(unix)] assert_eq!( service .get(serde_yaml::Value::String("user".to_string())) .and_then(serde_yaml::Value::as_str), - expected_user.as_deref(), - "compose site controller should use the host uid/gid on Linux so bind-mounted runtime \ - state stays deletable and fall back to root on macOS where Docker Desktop mediates the \ - socket mount" + Some("0:0"), + "compose site controller must stay root so it can access the router-control volume, which \ + compose initializes as a root-owned mount point for the router runtime" ); let volumes = service .get(serde_yaml::Value::String("volumes".to_string())) @@ -9243,29 +9224,13 @@ volumes: .iter() .any(|value| { value.as_str() == Some("/var/run/docker.sock:/var/run/docker.sock") }) ); - #[cfg(all(unix, not(target_os = "macos")))] - if let Ok(metadata) = fs::metadata("/var/run/docker.sock") { - let expected_gid = metadata.gid().to_string(); - let group_add = service + assert!( + service .get(serde_yaml::Value::String("group_add".to_string())) - .and_then(serde_yaml::Value::as_sequence) - .expect("controller service should declare supplemental groups"); - if expected_gid != unsafe { libc::getegid() }.to_string() { - assert!( - group_add - .iter() - .any(|value| value.as_str() == Some(expected_gid.as_str())), - "compose site controller should join the host docker socket group when the socket \ - gid differs from the host gid", - ); - } else { - assert!( - group_add.is_empty(), - "compose site controller should not add a redundant supplemental group when the \ - host gid already owns the docker socket", - ); - } - } + .is_none(), + "compose site controller should not override its supplemental groups; it runs as root to \ + preserve access to both the Docker socket and the router-control volume" + ); assert!( extra_hosts .iter() From 4b6ae3c2b876ebc2035c582ee9db29f975a4d3b3 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Fri, 17 Apr 2026 09:29:13 +0800 Subject: [PATCH 08/25] fix tests --- cli/src/mixed_run.rs | 17 +- cli/src/mixed_run/supervisor.rs | 20 +- cli/src/mixed_run/tests.rs | 39 +- cli/src/tests.rs | 26 +- cli/src/vm_runtime.rs | 10 +- cli/tests/linux_vm.rs | 77 +- cli/tests/mixed_run/smoke.rs | 251 ++++--- cli/tests/mixed_run/support.rs | 668 ++++++++++++++++-- .../site-controller/src/default_runtime.rs | 46 +- .../src/default_runtime/child_runtime.rs | 82 ++- .../default_runtime/site_runtime_support.rs | 59 +- runtime/site-controller/src/lib.rs | 3 +- 12 files changed, 1040 insertions(+), 258 deletions(-) diff --git a/cli/src/mixed_run.rs b/cli/src/mixed_run.rs index d480e127..87b7e789 100644 --- a/cli/src/mixed_run.rs +++ b/cli/src/mixed_run.rs @@ -1241,7 +1241,15 @@ fn kubectl_list_namespace_pods_with_bin( namespace: &str, ) -> Result> { let output = kubectl_command_with_bin(kubectl_bin, context) - .args(["-n", namespace, "get", "pods", "--ignore-not-found", "-o", "name"]) + .args([ + "-n", + namespace, + "get", + "pods", + "--ignore-not-found", + "-o", + "name", + ]) .output() .into_diagnostic() .wrap_err_with(|| { @@ -1288,7 +1296,9 @@ fn stop_kubernetes_namespace_with_kubectl( ]) .output() .into_diagnostic() - .wrap_err_with(|| format!("failed to start deletion of kubernetes namespace `{namespace}`"))?; + .wrap_err_with(|| { + format!("failed to start deletion of kubernetes namespace `{namespace}`") + })?; if !delete_output.status.success() && !kubectl_is_not_found(&delete_output) { return Err(miette::miette!( "starting deletion of kubernetes namespace `{namespace}` failed:\n{}", @@ -1355,7 +1365,8 @@ fn stop_kubernetes_namespace_with_kubectl( .map(|detail| format!("last namespace response:\n{detail}")) .unwrap_or_else(|| "last namespace response: ".to_string()); return Err(miette::miette!( - "timed out waiting for kubernetes namespace `{namespace}` to delete after {}s\n{pod_detail}\n{namespace_detail}", + "timed out waiting for kubernetes namespace `{namespace}` to delete after \ + {}s\n{pod_detail}\n{namespace_detail}", timeout.as_secs() + KUBERNETES_NAMESPACE_FORCE_DELETE_GRACE_PERIOD.as_secs() )); } diff --git a/cli/src/mixed_run/supervisor.rs b/cli/src/mixed_run/supervisor.rs index 1a35b3a7..e7728719 100644 --- a/cli/src/mixed_run/supervisor.rs +++ b/cli/src/mixed_run/supervisor.rs @@ -1067,15 +1067,17 @@ fn compose_site_timeout_sections(plan: &SiteSupervisorPlan) -> Vec { .unwrap_or_else(|| "unavailable".to_string()); sections.push(format!( "compose site controller status (`{}`): {summary}", - compose_site_controller_container_name(plan) - .unwrap_or_else(|| "".to_string()) + compose_site_controller_container_name(plan).unwrap_or_else(|| "".to_string()) )); } if let Some(compose_ps) = timeout_command_section( "docker compose ps --all", - compose_command(plan.compose_project.as_deref(), Path::new(&plan.artifact_dir)) - .arg("ps") - .arg("--all"), + compose_command( + plan.compose_project.as_deref(), + Path::new(&plan.artifact_dir), + ) + .arg("ps") + .arg("--all"), ) { sections.push(compose_ps); } @@ -1113,9 +1115,11 @@ fn site_startup_timeout_diagnostics( if let Some(section) = timeout_file_section("manager state", state_path, 200) { sections.push(section); } - if let Some(section) = - timeout_file_section("supervisor log", &site_state_root.join("supervisor.log"), 200) - { + if let Some(section) = timeout_file_section( + "supervisor log", + &site_state_root.join("supervisor.log"), + 200, + ) { sections.push(section); } if let Some(section) = timeout_file_section("site log", &site_state_root.join("site.log"), 200) diff --git a/cli/src/mixed_run/tests.rs b/cli/src/mixed_run/tests.rs index fb7d5ac0..d617e91a 100644 --- a/cli/src/mixed_run/tests.rs +++ b/cli/src/mixed_run/tests.rs @@ -1,3 +1,5 @@ +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt as _; use std::{ fs, net::{SocketAddr, TcpListener}, @@ -9,9 +11,6 @@ use tempfile::TempDir; use super::*; -#[cfg(unix)] -use std::os::unix::fs::PermissionsExt as _; - #[test] fn site_state_paths_are_site_scoped() { let root = Path::new("/tmp/amber-run/state"); @@ -181,8 +180,8 @@ fn provisioner_image_supports_debug_builds() { .expect("provisioner Dockerfile should read"); assert!( dockerfile.contains("ARG BUILD_MODE=release"), - "provisioner image should accept the shared BUILD_MODE argument so mixed-run tests can use \ - debug builds:\n{dockerfile}" + "provisioner image should accept the shared BUILD_MODE argument so mixed-run tests can \ + use debug builds:\n{dockerfile}" ); assert!( dockerfile.contains("if [ \"$BUILD_MODE\" = \"release\" ]; then") @@ -410,16 +409,17 @@ fn stop_kubernetes_namespace_force_deletes_stuck_pods_before_retrying() { &kubectl, format!( "#!/bin/sh\nset -eu\nlog_path='{}'\nstate_path='{}'\nprintf '%s\\n' \"$*\" >> \ - \"$log_path\"\nif [ \"${{1:-}}\" = \"--context\" ]; then\n shift 2\nfi\nstate=alive\n\ - if [ -f \"$state_path\" ]; then\n state=$(cat \"$state_path\")\nfi\nif [ \"${{1:-}}\" \ - = \"delete\" ] && [ \"${{2:-}}\" = \"namespace\" ]; then\n exit 0\nfi\nif [ \ - \"${{1:-}}\" = \"get\" ] && [ \"${{2:-}}\" = \"namespace\" ]; then\n if [ \"$state\" = \ - \"gone\" ]; then\n echo 'Error from server (NotFound): namespaces \"'\"${{3:-}}\"'\" \ - not found' >&2\n exit 1\n fi\n printf '{{\"metadata\":{{\"name\":\"%s\",\ - \"deletionTimestamp\":\"2026-04-16T02:33:55Z\"}}}}\\n' \"${{3:-}}\"\n exit 0\nfi\nif \ - [ \"${{1:-}}\" = \"-n\" ] && [ \"${{2:-}}\" = \"test-ns\" ] && [ \"${{3:-}}\" = \ - \"delete\" ] && [ \"${{4:-}}\" = \"pods\" ]; then\n printf 'gone' > \"$state_path\"\n \ - exit 0\nfi\necho \"unexpected kubectl invocation: $*\" >&2\nexit 1\n", + \"$log_path\"\nif [ \"${{1:-}}\" = \"--context\" ]; then\n shift \ + 2\nfi\nstate=alive\nif [ -f \"$state_path\" ]; then\n state=$(cat \ + \"$state_path\")\nfi\nif [ \"${{1:-}}\" = \"delete\" ] && [ \"${{2:-}}\" = \ + \"namespace\" ]; then\n exit 0\nfi\nif [ \"${{1:-}}\" = \"get\" ] && [ \"${{2:-}}\" \ + = \"namespace\" ]; then\n if [ \"$state\" = \"gone\" ]; then\n echo 'Error from \ + server (NotFound): namespaces \"'\"${{3:-}}\"'\" not found' >&2\n exit 1\n fi\n \ + printf '{{\"metadata\":{{\"name\":\"%s\",\"deletionTimestamp\":\"2026-04-16T02:33:\ + 55Z\"}}}}\\n' \"${{3:-}}\"\n exit 0\nfi\nif [ \"${{1:-}}\" = \"-n\" ] && [ \ + \"${{2:-}}\" = \"test-ns\" ] && [ \"${{3:-}}\" = \"delete\" ] && [ \"${{4:-}}\" = \ + \"pods\" ]; then\n printf 'gone' > \"$state_path\"\n exit 0\nfi\necho \"unexpected \ + kubectl invocation: $*\" >&2\nexit 1\n", log_path.display(), state_path.display(), ), @@ -442,9 +442,7 @@ fn stop_kubernetes_namespace_force_deletes_stuck_pods_before_retrying() { let log = fs::read_to_string(&log_path).expect("kubectl log"); let delete_namespace_calls = log .lines() - .filter(|line| { - line.contains("delete namespace test-ns --ignore-not-found --wait=false") - }) + .filter(|line| line.contains("delete namespace test-ns --ignore-not-found --wait=false")) .count(); assert_eq!( delete_namespace_calls, 2, @@ -452,8 +450,9 @@ fn stop_kubernetes_namespace_force_deletes_stuck_pods_before_retrying() { ); assert!( log.lines().any(|line| { - line.contains("-n test-ns delete pods --all --ignore-not-found --force \ - --grace-period=0") + line.contains( + "-n test-ns delete pods --all --ignore-not-found --force --grace-period=0", + ) }), "forced pod cleanup should run before the retry:\n{log}" ); diff --git a/cli/src/tests.rs b/cli/src/tests.rs index c1b23033..8460cd0c 100644 --- a/cli/src/tests.rs +++ b/cli/src/tests.rs @@ -942,14 +942,26 @@ child.wait() } let deadline = Instant::now() + Duration::from_secs(5); - while !child_pid_path.is_file() && Instant::now() < deadline { + let child_pid = loop { + if Instant::now() >= deadline { + let contents = fs::read_to_string(&child_pid_path).unwrap_or_default(); + panic!( + "descendant pid should be recorded before cleanup starts; last contents: {:?}", + contents, + ); + } + match fs::read_to_string(&child_pid_path) { + Ok(contents) => { + let trimmed = contents.trim(); + if let Ok(pid) = trimmed.parse::() { + break pid; + } + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => panic!("descendant pid file should be readable: {err}"), + } sleep(Duration::from_millis(25)).await; - } - let child_pid = fs::read_to_string(&child_pid_path) - .expect("descendant pid should be recorded") - .trim() - .parse::() - .expect("descendant pid should parse"); + }; let wrapper_pid = children[0] .wrapper .as_ref() diff --git a/cli/src/vm_runtime.rs b/cli/src/vm_runtime.rs index 8ab62cd2..95cce33e 100644 --- a/cli/src/vm_runtime.rs +++ b/cli/src/vm_runtime.rs @@ -797,13 +797,13 @@ fn resolve_vm_runtime_host_path(raw_path: &str, source_dir: Option<&str>) -> Res } pub(crate) fn vm_uses_tcg_accel() -> bool { - matches!(detect_qemu_accel(), QemuAccel::Tcg) + amber_site_controller::vm_uses_tcg_accel() } fn detect_qemu_accel() -> QemuAccel { #[cfg(target_os = "macos")] { - if env::var_os("AMBER_VM_FORCE_TCG").is_some() { + if vm_uses_tcg_accel() { QemuAccel::Tcg } else { QemuAccel::Hvf @@ -812,12 +812,10 @@ fn detect_qemu_accel() -> QemuAccel { #[cfg(target_os = "linux")] { - if env::var_os("AMBER_VM_FORCE_TCG").is_some() { + if vm_uses_tcg_accel() { QemuAccel::Tcg - } else if Path::new("/dev/kvm").exists() { - QemuAccel::Kvm } else { - QemuAccel::Tcg + QemuAccel::Kvm } } diff --git a/cli/tests/linux_vm.rs b/cli/tests/linux_vm.rs index 9e43932e..34a911eb 100644 --- a/cli/tests/linux_vm.rs +++ b/cli/tests/linux_vm.rs @@ -433,6 +433,33 @@ fn write_ssh_keypair(private_key: &Path) -> Result { }) } +fn append_ssh_transport_args( + command: &mut Command, + port_flag: &str, + ssh_private_key: &Path, + ssh_port: u16, +) { + command + .arg("-i") + .arg(ssh_private_key) + .arg(port_flag) + .arg(ssh_port.to_string()) + .arg("-o") + .arg("BatchMode=yes") + .arg("-o") + .arg("StrictHostKeyChecking=no") + .arg("-o") + .arg("UserKnownHostsFile=/dev/null") + .arg("-o") + .arg("ConnectTimeout=5") + .arg("-o") + .arg("ServerAliveInterval=15") + .arg("-o") + .arg("ServerAliveCountMax=4") + .arg("-o") + .arg("LogLevel=ERROR"); +} + impl LinuxVmHarness { fn start(output_dir: &Path, base_image: &Path) -> Result { let arch = guest_arch(); @@ -573,18 +600,8 @@ impl LinuxVmHarness { fn ssh_base_command(&self) -> Command { let mut command = Command::new("/usr/bin/ssh"); - command - .arg("-i") - .arg(&self.ssh_private_key) - .arg("-p") - .arg(self.ssh_port.to_string()) - .arg("-o") - .arg("StrictHostKeyChecking=no") - .arg("-o") - .arg("UserKnownHostsFile=/dev/null") - .arg("-o") - .arg("LogLevel=ERROR") - .arg(format!("{GUEST_USER}@127.0.0.1")); + append_ssh_transport_args(&mut command, "-p", &self.ssh_private_key, self.ssh_port); + command.arg(format!("{GUEST_USER}@127.0.0.1")); command } @@ -634,17 +651,7 @@ impl LinuxVmHarness { fn scp_to_guest(&self, sources: &[&Path], description: &str) -> Result<(), String> { let mut command = Command::new("/usr/bin/scp"); - command - .arg("-i") - .arg(&self.ssh_private_key) - .arg("-P") - .arg(self.ssh_port.to_string()) - .arg("-o") - .arg("StrictHostKeyChecking=no") - .arg("-o") - .arg("UserKnownHostsFile=/dev/null") - .arg("-o") - .arg("LogLevel=ERROR"); + append_ssh_transport_args(&mut command, "-P", &self.ssh_private_key, self.ssh_port); for source in sources { command.arg(source); } @@ -1060,3 +1067,27 @@ fn provisioned_cache_keys_track_guest_setup() { assert!(mixed_run_cache_key.starts_with("mixed-run-")); assert_ne!(vm_smoke_cache_key, mixed_run_cache_key); } + +#[test] +fn ssh_transport_args_include_fail_fast_liveness_bounds() { + let mut command = Command::new("/usr/bin/ssh"); + append_ssh_transport_args(&mut command, "-p", Path::new("/tmp/test-key"), 2222); + let args = command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + for expected in [ + ["-o", "BatchMode=yes"], + ["-o", "ConnectTimeout=5"], + ["-o", "ServerAliveInterval=15"], + ["-o", "ServerAliveCountMax=4"], + ] { + assert!( + args.windows(2) + .any(|window| window[0] == expected[0] && window[1] == expected[1]), + "ssh transport args should include {} {} so dead guests fail fast: {args:?}", + expected[0], + expected[1], + ); + } +} diff --git a/cli/tests/mixed_run/smoke.rs b/cli/tests/mixed_run/smoke.rs index 02e82260..e2b1c5b7 100644 --- a/cli/tests/mixed_run/smoke.rs +++ b/cli/tests/mixed_run/smoke.rs @@ -1,6 +1,28 @@ use super::*; -const FRAMEWORK_MUTATION_REQUEST_TIMEOUT: Duration = Duration::from_secs(300); +const FRAMEWORK_CONTROL_RESPONSE_GRACE: Duration = Duration::from_secs(30); + +fn framework_control_timeout_for_vm_ready_timeout(vm_ready_timeout: Duration) -> Duration { + vm_ready_timeout.max(Duration::from_secs(300)) +} + +fn framework_control_timeout() -> Duration { + framework_control_timeout_for_vm_ready_timeout( + amber_site_controller::vm_endpoint_forward_ready_timeout(), + ) +} + +fn framework_mutation_request_timeout_for_control_timeout(control_timeout: Duration) -> Duration { + control_timeout + FRAMEWORK_CONTROL_RESPONSE_GRACE +} + +fn framework_mutation_request_timeout() -> Duration { + framework_mutation_request_timeout_for_control_timeout(framework_control_timeout()) +} + +fn framework_control_timeout_env_value() -> String { + format!("{:.3}", framework_control_timeout().as_secs_f64()) +} const FRAMEWORK_ADMIN_APP: &str = r#"import json import os @@ -11,7 +33,7 @@ from urllib.request import Request, urlopen NAME = os.environ["NAME"] PORT = int(os.environ["PORT"]) CTL_URL = os.environ["CTL_URL"].rstrip("/") -CONTROL_TIMEOUT = 300.0 +CONTROL_TIMEOUT = float(os.environ.get("CONTROL_TIMEOUT", "300.0")) def send(handler, status, body, content_type="text/plain; charset=utf-8"): payload = body.encode("utf-8") @@ -567,12 +589,13 @@ ThreadingHTTPServer(("0.0.0.0", PORT), Handler).serve_forever() fn write_framework_admin_component(root: &Path, file_name: &str, image: bool, port: u16) { let program = if image { json!({ - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["python3", "-u", "-c", { "file": "./admin.py" }], "env": { "NAME": "admin", "PORT": port.to_string(), - "CTL_URL": "${slots.ctl.url}" + "CTL_URL": "${slots.ctl.url}", + "CONTROL_TIMEOUT": framework_control_timeout_env_value() }, "network": { "endpoints": [ @@ -587,7 +610,8 @@ fn write_framework_admin_component(root: &Path, file_name: &str, image: bool, po "env": { "NAME": "admin", "PORT": port.to_string(), - "CTL_URL": "${slots.ctl.url}" + "CTL_URL": "${slots.ctl.url}", + "CONTROL_TIMEOUT": framework_control_timeout_env_value() }, "network": { "endpoints": [ @@ -623,7 +647,7 @@ fn write_framework_worker_component( ) { let program = if image { json!({ - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["python3", "-u", "-c", { "file": "./worker.py" }], "env": { "NAME": name, @@ -701,7 +725,7 @@ fn write_dynamic_caps_component( .collect::>(); let program = if image { json!({ - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["python3", "-u", "-c", { "file": "./dynamic_caps_app.py" }], "env": env, "network": { @@ -947,6 +971,7 @@ write_files: Environment=NAME=admin Environment=PORT={port} Environment=CTL_URL=${{slots.ctl.url}} + Environment=CONTROL_TIMEOUT={control_timeout} ExecStart=/usr/bin/python3 /usr/local/bin/framework-admin.py Restart=always @@ -957,6 +982,7 @@ runcmd: - [systemctl, enable, --now, framework-admin.service] "#, script = indent_block(FRAMEWORK_ADMIN_APP, 6), + control_timeout = framework_control_timeout_env_value(), ) } @@ -1417,42 +1443,78 @@ fn assert_string_array_members(value: &Value, expected: &[&str], message: &str) assert_eq!(actual, expected, "{message}"); } +fn framework_admin_request( + method: &str, + port: u16, + path: &str, + body: Option<&str>, + purpose: &str, +) -> (u16, String) { + let timeout = framework_mutation_request_timeout(); + http_request_with_timeout_result(method, port, path, body, timeout).unwrap_or_else(|err| { + panic!("{purpose} via framework admin {method} http://127.0.0.1:{port}{path} failed: {err}") + }) +} + +fn framework_admin_get(port: u16, path: &str, purpose: &str) -> (u16, String) { + let timeout = framework_mutation_request_timeout(); + http_get_with_timeout_result(port, path, timeout).unwrap_or_else(|err| { + panic!("{purpose} via framework admin GET http://127.0.0.1:{port}{path} failed: {err}") + }) +} + fn framework_create_child_with_request(port: u16, request: &Value) -> (u16, String) { let body = serde_json::to_string(request).expect("create request should serialize"); - http_request_with_timeout( - "POST", - port, - "/create", - Some(&body), - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("create request should return an HTTP response") + framework_admin_request("POST", port, "/create", Some(&body), "create request") } fn framework_destroy_child_via_admin(port: u16, name: &str) -> (u16, String) { - http_request_with_timeout( + framework_admin_request( "DELETE", port, &format!("/destroy/{name}"), None, - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, + "destroy request", ) - .expect("destroy request should return an HTTP response") } fn framework_snapshot_via_admin(port: u16) -> Value { - let (status, body) = http_request_with_timeout( - "POST", - port, - "/snapshot", - Some("{}"), - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("snapshot request should return an HTTP response"); + let (status, body) = + framework_admin_request("POST", port, "/snapshot", Some("{}"), "snapshot request"); assert_eq!(status, 200, "snapshot request should succeed: {body}"); serde_json::from_str(&body).expect("snapshot response should be valid json") } +#[test] +fn framework_control_timeout_tracks_vm_ready_budget() { + assert_eq!( + framework_control_timeout_for_vm_ready_timeout(Duration::from_secs(120)), + Duration::from_secs(300), + "fast VM environments should keep the existing 300s framework control budget", + ); + assert_eq!( + framework_control_timeout_for_vm_ready_timeout(Duration::from_secs(720)), + Duration::from_secs(720), + "slow TCG VM environments should inherit the full VM-ready budget", + ); +} + +#[test] +fn framework_mutation_request_timeout_keeps_response_grace_after_control_budget() { + assert_eq!( + framework_mutation_request_timeout_for_control_timeout(Duration::from_secs(300)), + Duration::from_secs(330), + "framework admin curl requests should outlive the inner control timeout long enough to \ + receive the terminal response", + ); + assert_eq!( + framework_mutation_request_timeout_for_control_timeout(Duration::from_secs(720)), + Duration::from_secs(750), + "slow TCG control paths should keep the same response grace instead of truncating the \ + request at 300s", + ); +} + fn write_snapshot_run_inputs(root: &Path, snapshot: &Value) -> (PathBuf, PathBuf) { let scenario_path = root.join("snapshot-scenario.json"); let placement_path = root.join("snapshot-placement.json5"); @@ -1964,7 +2026,7 @@ fn framework_component_concurrent_create_serialization_live() { "required": ["name"] }, "program": { - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["python3", "-u", "-c", { "file": "./worker.py" }], "env": { "NAME": "${config.name}", @@ -2324,7 +2386,7 @@ fn framework_component_root_external_binding_live() { "catalog_api": { "kind": "http" } }, "program": { - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["python3", "-u", "-c", { "file": "./external_bind.py" }], "env": { "NAME": "external-worker", @@ -2379,7 +2441,7 @@ fn framework_component_root_external_binding_live() { "catalog_api": { "kind": "http" } }, "program": { - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["sleep", "3600"] }, "components": { @@ -2580,7 +2642,7 @@ fn framework_component_nonweak_publication_barrier_live() { "weak_api": { "kind": "http" } }, "program": { - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["python3", "-u", "-c", { "file": "./barrier_probe.py" }], "env": { "NAME": "consumer", @@ -2612,7 +2674,7 @@ fn framework_component_nonweak_publication_barrier_live() { "delayed_api": { "kind": "http" } }, "program": { - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["sleep", "3600"] }, "components": { @@ -3100,12 +3162,11 @@ fn framework_component_compose_parent_standby_direct_live() { "admin" ); - let (create_status, create_response) = http_get_with_timeout( + let (create_status, create_response) = framework_admin_get( proxy_port, "/create/worker/job-1", - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("create request should return an HTTP response"); + "create request should return an HTTP response", + ); assert_eq!(create_status, 200, "create request should succeed"); let create_json: Value = serde_json::from_str(&create_response).expect("create response should be valid json"); @@ -3239,12 +3300,11 @@ fn framework_component_direct_parent_compose_child_live() { "admin" ); - let (create_status, create_response) = http_get_with_timeout( + let (create_status, create_response) = framework_admin_get( proxy_port, "/create/job-1", - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("create request should return an HTTP response"); + "create request should return an HTTP response", + ); assert_eq!(create_status, 200, "create request should succeed"); let create_json: Value = serde_json::from_str(&create_response).expect("create response should be valid json"); @@ -3373,12 +3433,11 @@ fn framework_component_dynamic_children_teardown_with_run_live() { ); wait_for_path(&mut admin_proxy, proxy_port, "/id", Duration::from_secs(60)); - let (create_direct_status, create_direct_response) = http_get_with_timeout( + let (create_direct_status, create_direct_response) = framework_admin_get( proxy_port, "/create/worker_direct/job-direct", - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("direct create request should return an HTTP response"); + "direct create request should return an HTTP response", + ); assert_eq!( create_direct_status, 200, "direct child create request should succeed" @@ -3387,12 +3446,11 @@ fn framework_component_dynamic_children_teardown_with_run_live() { .expect("direct child create response should be valid json"); assert_eq!(create_direct_json["child"]["name"], "job-direct"); - let (create_compose_status, create_compose_response) = http_get_with_timeout( + let (create_compose_status, create_compose_response) = framework_admin_get( proxy_port, "/create/worker_compose/job-compose", - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("compose create request should return an HTTP response"); + "compose create request should return an HTTP response", + ); assert_eq!( create_compose_status, 200, "compose child create request should succeed" @@ -3576,12 +3634,11 @@ fn framework_component_destroy_of_provider_keeps_consumer_live() { ); wait_for_path(&mut admin_proxy, proxy_port, "/id", Duration::from_secs(60)); - let (create_provider_status, create_provider_response) = http_get_with_timeout( + let (create_provider_status, create_provider_response) = framework_admin_get( proxy_port, "/create/producer/source", - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("producer create request should return an HTTP response"); + "producer create request should return an HTTP response", + ); assert_eq!( create_provider_status, 200, "producer create request should succeed" @@ -3732,7 +3789,7 @@ fn framework_component_kind_root_export_live() { &run.run_root, "vm_local", "running", - Duration::from_secs(240), + framework_control_timeout(), ); let site_state = |site_id: &str| match site_id { "compose_local" => &compose_state, @@ -3754,7 +3811,7 @@ fn framework_component_kind_root_export_live() { &mut creator_proxy, creator_port, "/id", - Duration::from_secs(240), + framework_mutation_request_timeout(), ); assert_eq!( wait_for_body( @@ -3766,12 +3823,11 @@ fn framework_component_kind_root_export_live() { "admin", ); - let (create_status, create_response) = http_get_with_timeout( + let (create_status, create_response) = framework_admin_get( creator_port, "/create/child_kind/job-kind", - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("create request should return an HTTP response"); + "create request should return an HTTP response", + ); assert_eq!( create_status, 200, "create request should succeed; response: {create_response}" @@ -3782,7 +3838,12 @@ fn framework_component_kind_root_export_live() { let root_port = pick_free_port(); let mut root_proxy = spawn_framework_proxy_for_site(&root_artifact, "http", root_port, site_state("kind_local")); - wait_for_path(&mut root_proxy, root_port, "/id", Duration::from_secs(300)); + wait_for_path( + &mut root_proxy, + root_port, + "/id", + framework_mutation_request_timeout(), + ); assert_eq!( wait_for_body(&mut root_proxy, root_port, "/id", Duration::from_secs(30)), "child-kind-root" @@ -3835,7 +3896,7 @@ fn framework_component_kind_creator_compose_child_live() { &run.run_root, "vm_local", "running", - Duration::from_secs(240), + framework_control_timeout(), ); let site_state = |site_id: &str| match site_id { "compose_local" => &compose_state, @@ -3857,7 +3918,7 @@ fn framework_component_kind_creator_compose_child_live() { &mut creator_proxy, creator_port, "/id", - Duration::from_secs(240), + framework_mutation_request_timeout(), ); assert_eq!( wait_for_body( @@ -3870,12 +3931,11 @@ fn framework_component_kind_creator_compose_child_live() { "kind creator should expose the framework admin app" ); - let (create_status, create_response) = http_get_with_timeout( + let (create_status, create_response) = framework_admin_get( creator_port, "/create/child_compose/job-compose", - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("create request should return an HTTP response"); + "create request should return an HTTP response", + ); assert_eq!( create_status, 200, "create request should succeed; response: {create_response}" @@ -3890,7 +3950,12 @@ fn framework_component_kind_creator_compose_child_live() { root_port, site_state("compose_local"), ); - wait_for_path(&mut root_proxy, root_port, "/id", Duration::from_secs(300)); + wait_for_path( + &mut root_proxy, + root_port, + "/id", + framework_mutation_request_timeout(), + ); assert_eq!( wait_for_body(&mut root_proxy, root_port, "/id", Duration::from_secs(30)), "child-compose-root" @@ -3943,7 +4008,7 @@ fn framework_component_kind_creator_after_compose_churn_live() { &run.run_root, "vm_local", "running", - Duration::from_secs(240), + framework_control_timeout(), ); let site_state = |site_id: &str| match site_id { "compose_local" => &compose_state, @@ -3966,7 +4031,7 @@ fn framework_component_kind_creator_after_compose_churn_live() { &mut compose_creator_proxy, compose_creator_port, "/id", - Duration::from_secs(240), + framework_mutation_request_timeout(), ); assert_eq!( wait_for_body( @@ -3984,12 +4049,11 @@ fn framework_component_kind_creator_after_compose_churn_live() { "/create/{}/{}", template_case.template, template_case.child_name ); - let (create_status, create_response) = http_get_with_timeout( + let (create_status, create_response) = framework_admin_get( compose_creator_port, &create_path, - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("compose churn create request should return an HTTP response"); + "compose churn create request should return an HTTP response", + ); assert_eq!( create_status, 200, "compose churn create request {create_path} should succeed; response: \ @@ -4014,7 +4078,7 @@ fn framework_component_kind_creator_after_compose_churn_live() { template_case.child_name, &child_sites, child_id, - Duration::from_secs(300), + framework_mutation_request_timeout(), ); } stop_proxy(&mut compose_creator_proxy); @@ -4030,7 +4094,7 @@ fn framework_component_kind_creator_after_compose_churn_live() { &mut kind_creator_proxy, kind_creator_port, "/id", - Duration::from_secs(240), + framework_mutation_request_timeout(), ); assert_eq!( wait_for_body( @@ -4056,12 +4120,11 @@ fn framework_component_kind_creator_after_compose_churn_live() { "kind creator control path should be healthy after compose churn" ); - let (create_status, create_response) = http_get_with_timeout( + let (create_status, create_response) = framework_admin_get( kind_creator_port, "/create/child_compose/job-compose", - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("kind create request should return an HTTP response"); + "kind create request should return an HTTP response", + ); assert_eq!( create_status, 200, "kind creator create after compose churn should succeed; response: {create_response}" @@ -4076,7 +4139,12 @@ fn framework_component_kind_creator_after_compose_churn_live() { root_port, site_state("compose_local"), ); - wait_for_path(&mut root_proxy, root_port, "/id", Duration::from_secs(300)); + wait_for_path( + &mut root_proxy, + root_port, + "/id", + framework_mutation_request_timeout(), + ); assert_eq!( wait_for_body(&mut root_proxy, root_port, "/id", Duration::from_secs(30)), "child-compose-root", @@ -4130,7 +4198,7 @@ fn framework_component_cross_backend_matrix_live() { &run.run_root, "vm_local", "running", - Duration::from_secs(240), + framework_control_timeout(), ); let site_state = |site_id: &str| match site_id { "compose_local" => &compose_state, @@ -4154,7 +4222,7 @@ fn framework_component_cross_backend_matrix_live() { &mut creator_proxy, creator_port, "/id", - Duration::from_secs(240), + framework_mutation_request_timeout(), ); assert_eq!( wait_for_body( @@ -4172,12 +4240,11 @@ fn framework_component_cross_backend_matrix_live() { "/create/{}/{}", template_case.template, template_case.child_name ); - let (create_status, create_response) = http_get_with_timeout( + let (create_status, create_response) = framework_admin_get( creator_port, &create_path, - FRAMEWORK_MUTATION_REQUEST_TIMEOUT, - ) - .expect("create request should return an HTTP response"); + "create request should return an HTTP response", + ); assert_eq!( create_status, 200, "create request {create_path} from {creator_site} should succeed; response: \ @@ -4203,7 +4270,12 @@ fn framework_component_cross_backend_matrix_live() { root_port, site_state(root_site_id), ); - wait_for_path(&mut root_proxy, root_port, "/id", Duration::from_secs(300)); + wait_for_path( + &mut root_proxy, + root_port, + "/id", + framework_mutation_request_timeout(), + ); assert_eq!( wait_for_body(&mut root_proxy, root_port, "/id", Duration::from_secs(30)), template_case.exports[0].2 @@ -4218,7 +4290,12 @@ fn framework_component_cross_backend_matrix_live() { port, site_state(site_id), ); - wait_for_path(&mut proxy, port, "/id", Duration::from_secs(300)); + wait_for_path( + &mut proxy, + port, + "/id", + framework_mutation_request_timeout(), + ); assert_eq!( wait_for_body(&mut proxy, port, "/id", Duration::from_secs(30)), *expected_id, @@ -4251,7 +4328,7 @@ fn framework_component_cross_backend_matrix_live() { template_case.child_name, &child_sites, child_id, - Duration::from_secs(300), + framework_mutation_request_timeout(), ); } diff --git a/cli/tests/mixed_run/support.rs b/cli/tests/mixed_run/support.rs index 568421db..e26409c0 100644 --- a/cli/tests/mixed_run/support.rs +++ b/cli/tests/mixed_run/support.rs @@ -138,7 +138,14 @@ class Handler(BaseHTTPRequestHandler): ThreadingHTTPServer(("0.0.0.0", PORT), Handler).serve_forever() "#; -pub(crate) const TEST_APP_IMAGE: &str = "python:3.13-alpine"; +const TEST_APP_SOURCE_IMAGE: &str = "python:3.13-alpine"; +const TEST_APP_LOCAL_IMAGE_REPOSITORY: &str = "amber-mixed-run-test-app"; + +#[derive(Clone, Debug, PartialEq, Eq)] +struct DockerImageMeta { + id: String, + arch: String, +} pub(crate) fn outputs_root() -> PathBuf { cli_test_outputs_root(&workspace_root()) @@ -331,7 +338,7 @@ pub(crate) fn docker_host_ip() -> String { .arg("host.docker.internal:host-gateway"); } let output = cmd - .arg(TEST_APP_IMAGE) + .arg(test_app_image()) .arg("python3") .arg("-c") .arg("import socket; print(socket.gethostbyname('host.docker.internal'))") @@ -346,22 +353,137 @@ pub(crate) fn docker_host_ip() -> String { String::from_utf8_lossy(&output.stdout).trim().to_string() } -pub(crate) fn ensure_local_image(tag: &str) { - let inspect = Command::new("docker") +fn docker_platform_arch() -> &'static str { + match env::consts::ARCH { + "aarch64" => "arm64", + "x86_64" => "amd64", + other => panic!("mixed-run tests support only aarch64 and x86_64 hosts, found {other}"), + } +} + +fn docker_platform(expected_arch: &str) -> String { + format!("linux/{expected_arch}") +} + +fn docker_image_meta_with_binaries( + docker_bin: &Path, + tag: &str, +) -> Result, String> { + let output = Command::new(docker_bin) .arg("image") .arg("inspect") + .arg("-f") + .arg("{{.Id}}|{{.Architecture}}") .arg(tag) - .status() - .unwrap_or_else(|err| panic!("failed to inspect docker image {tag}: {err}")); - if inspect.success() { - return; + .output() + .map_err(|err| format!("failed to inspect docker image {tag}: {err}"))?; + if !output.status.success() { + return Ok(None); + } + let meta = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if meta.is_empty() { + return Ok(None); + } + let (id, arch) = meta.split_once('|').ok_or_else(|| { + format!("docker image inspect for {tag} returned malformed metadata: {meta}") + })?; + if id.is_empty() || arch.is_empty() { + return Err(format!( + "docker image inspect for {tag} returned incomplete metadata: {meta}" + )); + } + Ok(Some(DockerImageMeta { + id: id.to_string(), + arch: arch.to_string(), + })) +} + +fn ensure_local_image_with_binaries( + docker_bin: &Path, + tag: &str, + expected_arch: &str, +) -> Result { + if let Some(meta) = docker_image_meta_with_binaries(docker_bin, tag)? + && meta.arch == expected_arch + { + return Ok(meta); } - let status = Command::new("docker") + + let output = Command::new(docker_bin) .arg("pull") + .arg("--platform") + .arg(docker_platform(expected_arch)) .arg(tag) - .status() - .unwrap_or_else(|err| panic!("failed to pull docker image {tag}: {err}")); - assert!(status.success(), "docker pull failed for {tag}"); + .output() + .map_err(|err| format!("failed to pull docker image {tag}: {err}"))?; + if !output.status.success() { + return Err(format_command_output( + &format!("docker pull failed for {tag}"), + &output, + )); + } + + let meta = docker_image_meta_with_binaries(docker_bin, tag)? + .ok_or_else(|| format!("docker image {tag} is still unavailable after pull"))?; + if meta.arch != expected_arch { + return Err(format!( + "docker pull resolved {tag} to linux/{} instead of {}", + meta.arch, + docker_platform(expected_arch) + )); + } + Ok(meta) +} + +fn build_test_app_image_with_binaries( + docker_bin: &Path, + source_image: &str, + expected_arch: &str, +) -> Result { + let _ = ensure_local_image_with_binaries(docker_bin, source_image, expected_arch)?; + let local_tag = format!("{TEST_APP_LOCAL_IMAGE_REPOSITORY}:{expected_arch}"); + let build_root = tempfile::tempdir() + .map_err(|err| format!("failed to create temp dir for {local_tag}: {err}"))?; + fs::write( + build_root.path().join("Dockerfile"), + format!("FROM {source_image}\n"), + ) + .map_err(|err| { + format!( + "failed to write Dockerfile for test app image {local_tag} in {}: {err}", + build_root.path().display() + ) + })?; + let output = Command::new(docker_bin) + .arg("build") + .arg("--platform") + .arg(docker_platform(expected_arch)) + .arg("-t") + .arg(&local_tag) + .arg(build_root.path()) + .output() + .map_err(|err| format!("failed to build docker image {local_tag}: {err}"))?; + if !output.status.success() { + return Err(format_command_output( + &format!("docker build failed for {local_tag} from {source_image}"), + &output, + )); + } + Ok(local_tag) +} + +pub(crate) fn test_app_image() -> &'static str { + static IMAGE: OnceLock = OnceLock::new(); + IMAGE + .get_or_init(|| { + build_test_app_image_with_binaries( + Path::new("docker"), + TEST_APP_SOURCE_IMAGE, + docker_platform_arch(), + ) + .unwrap_or_else(|err| panic!("{err}")) + }) + .as_str() } pub(crate) fn docker_host_http_url(port: u16) -> String { @@ -395,6 +517,14 @@ pub(crate) fn http_get_with_timeout( http_request_with_timeout("GET", port, path, None, timeout) } +pub(crate) fn http_get_with_timeout_result( + port: u16, + path: &str, + timeout: Duration, +) -> Result<(u16, String), String> { + http_request_with_timeout_result("GET", port, path, None, timeout) +} + pub(crate) fn http_request_with_timeout( method: &str, port: u16, @@ -402,6 +532,16 @@ pub(crate) fn http_request_with_timeout( body: Option<&str>, timeout: Duration, ) -> Option<(u16, String)> { + http_request_with_timeout_result(method, port, path, body, timeout).ok() +} + +pub(crate) fn http_request_with_timeout_result( + method: &str, + port: u16, + path: &str, + body: Option<&str>, + timeout: Duration, +) -> Result<(u16, String), String> { let mut command = Command::new("curl"); command .arg("-sS") @@ -416,21 +556,46 @@ pub(crate) fn http_request_with_timeout( .arg("--data") .arg(body); } + let url = format!("http://127.0.0.1:{port}{path}"); let output = command_output_via_tempfiles( command .arg("-o") .arg("-") .arg("-w") .arg("\n%{http_code}") - .arg(format!("http://127.0.0.1:{port}{path}")), + .arg(&url), "curl http request", ); if !output.status.success() { - return None; + return Err(format_command_output( + &format!( + "curl {method} {url} failed after {:.3}s", + timeout.as_secs_f64() + ), + &output, + )); } let stdout = String::from_utf8_lossy(&output.stdout); - let (body, status) = stdout.rsplit_once('\n')?; - Some((status.trim().parse().ok()?, body.trim().to_string())) + let Some((body, status)) = stdout.rsplit_once('\n') else { + return Err(format!( + "curl {method} {url} returned a malformed response after \ + {:.3}s\nstdout:\n{}\nstderr:\n{}", + timeout.as_secs_f64(), + stdout, + String::from_utf8_lossy(&output.stderr), + )); + }; + let status = status.trim().parse().map_err(|err| { + format!( + "curl {method} {url} returned a malformed HTTP status `{}` after {:.3}s: \ + {err}\nstdout:\n{}\nstderr:\n{}", + status.trim(), + timeout.as_secs_f64(), + stdout, + String::from_utf8_lossy(&output.stderr), + ) + })?; + Ok((status, body.trim().to_string())) } pub(crate) fn wait_for_body( @@ -643,7 +808,9 @@ fn append_docker_project_debug(out: &mut String, project: &str) { return; } - out.push_str(&format!("\ndocker compose project `{project}` containers:\n")); + out.push_str(&format!( + "\ndocker compose project `{project}` containers:\n" + )); for (_, name, status) in &containers { out.push_str(&format!("{name}\t{status}\n")); } @@ -655,9 +822,7 @@ fn append_docker_project_debug(out: &mut String, project: &str) { { Ok(output) => output, Err(err) => { - out.push_str(&format!( - "\ndocker logs {name} ({status}) failed:\n{err}\n" - )); + out.push_str(&format!("\ndocker logs {name} ({status}) failed:\n{err}\n")); continue; } }; @@ -1017,7 +1182,7 @@ pub(crate) fn ensure_amber_internal_images() { pub(crate) fn ensure_internal_images() { static READY: OnceLock<()> = OnceLock::new(); READY.get_or_init(|| { - ensure_local_image(TEST_APP_IMAGE); + let _ = test_app_image(); ensure_amber_internal_images(); }); } @@ -1078,36 +1243,54 @@ fn ensure_local_dev_image_tag_overrides() { } pub(crate) fn load_kind_image(cluster_name: &str, image: &str) { - let mut last_error = None; + load_kind_image_with_binaries(Path::new("kind"), Path::new("docker"), cluster_name, image) + .unwrap_or_else(|err| panic!("{err}")); +} + +fn load_kind_image_with_binaries( + kind_bin: &Path, + docker_bin: &Path, + cluster_name: &str, + image: &str, +) -> Result<(), String> { + let mut direct_error = None; for attempt in 1..=3 { - let direct_error = match kind_load_docker_image(cluster_name, image) { - Ok(()) => return, - Err(err) => err, - }; - match kind_load_image_archive(cluster_name, image) { - Ok(()) => return, - Err(archive_error) => { - last_error = Some(format!( - "{direct_error}\n\nkind load image-archive fallback failed for {image} in \ - cluster {cluster_name}:\n{archive_error}" - )); + match kind_load_docker_image(kind_bin, cluster_name, image) { + Ok(()) => return Ok(()), + Err(err) => { + direct_error = Some(err); + if attempt < 3 { + eprintln!( + "kind load docker-image attempt {attempt} failed for {image} in cluster \ + {cluster_name}; retrying" + ); + thread::sleep(Duration::from_secs(attempt * 2)); + } } } - if attempt < 3 { - eprintln!( - "kind load docker-image attempt {attempt} failed for {image} in cluster \ - {cluster_name}; retrying" - ); - thread::sleep(Duration::from_secs(attempt * 2)); - } } - let error = last_error.expect("kind image load should record a failure"); - panic!("{error}"); + let direct_error = direct_error.expect("kind image load should record a failure"); + let archive = tempfile::Builder::new() + .prefix("amber-kind-image-") + .suffix(".tar") + .tempfile() + .map_err(|err| { + format!("failed to create temporary image archive for {image} in {cluster_name}: {err}") + })?; + kind_load_image_archive_with_binaries(kind_bin, docker_bin, cluster_name, image, archive) + .map_err(|archive_error| { + format!( + "{direct_error} + +kind load image-archive fallback failed for {image} in cluster {cluster_name}: +{archive_error}" + ) + }) } -fn kind_load_docker_image(cluster_name: &str, image: &str) -> Result<(), String> { - let output = Command::new("kind") +fn kind_load_docker_image(kind_bin: &Path, cluster_name: &str, image: &str) -> Result<(), String> { + let output = Command::new(kind_bin) .arg("load") .arg("docker-image") .arg("--name") @@ -1126,17 +1309,6 @@ fn kind_load_docker_image(cluster_name: &str, image: &str) -> Result<(), String> )) } -fn kind_load_image_archive(cluster_name: &str, image: &str) -> Result<(), String> { - let archive = tempfile::Builder::new() - .prefix("amber-kind-image-") - .suffix(".tar") - .tempfile() - .map_err(|err| { - format!("failed to create temporary image archive for {image} in {cluster_name}: {err}") - })?; - kind_load_image_archive_with_binaries(Path::new("kind"), Path::new("docker"), cluster_name, image, archive) -} - fn kind_load_image_archive_with_binaries( kind_bin: &Path, docker_bin: &Path, @@ -1203,7 +1375,11 @@ pub(crate) fn ensure_kind_internal_images(kind_cluster: &KindCluster) { let images = amber_internal_image_refs(); let ready_key = format!( "{name}|{}|{}|{}|{}|{}", - images.router, images.provisioner, images.helper, images.site_controller, TEST_APP_IMAGE + images.router, + images.provisioner, + images.helper, + images.site_controller, + test_app_image() ); let loaded = READY.get_or_init(|| Mutex::new(BTreeSet::new())); { @@ -1216,7 +1392,7 @@ pub(crate) fn ensure_kind_internal_images(kind_cluster: &KindCluster) { load_kind_image(&name, &images.provisioner); load_kind_image(&name, &images.helper); load_kind_image(&name, &images.site_controller); - load_kind_image(&name, TEST_APP_IMAGE); + load_kind_image(&name, test_app_image()); loaded .lock() .expect("kind image-load guard should lock") @@ -1958,7 +2134,7 @@ pub(crate) fn write_image_component( "manifest_version": "0.3.0", "slots": upstreams.iter().map(|(alias, _)| ((*alias).to_string(), json!({"kind": "http"}))).collect::>(), "program": { - "image": TEST_APP_IMAGE, + "image": test_app_image(), "entrypoint": ["python3", "-u", "-c", { "file": "./app.py" }], "env": env, "network": { @@ -3036,7 +3212,12 @@ mod tests { fs::write( &docker, format!( - "#!/bin/sh\nset -eu\nprintf 'docker %s\\n' \"$*\" >> '{}'\nif [ \"$1\" = image ] && [ \"$2\" = save ] && [ \"$3\" = --output ]; then\n printf 'archive' > \"$4\"\n exit 0\nfi\necho \"unexpected docker invocation: $*\" >&2\nexit 1\n", + "#!/bin/sh\nset -eu\nprintf 'docker %s\\n' \"$*\" >> '{}'\nif [ \"$1\" = image ] \ + && [ \"$2\" = inspect ] && [ \"$3\" = -f ] && [ \"$4\" = \ + '{{{{.Id}}}}|{{{{.Architecture}}}}' ] && [ \"$5\" = 'test:image' ]; then\n exit \ + 1\nfi\nif [ \"$1\" = image ] && [ \"$2\" = save ] && [ \"$3\" = --output ]; \ + then\n printf 'archive' > \"$4\"\n exit 0\nfi\necho \"unexpected docker \ + invocation: $*\" >&2\nexit 1\n", log_path.display() ), ) @@ -3044,7 +3225,10 @@ mod tests { fs::write( &kind, format!( - "#!/bin/sh\nset -eu\nprintf 'kind %s\\n' \"$*\" >> '{}'\narchive=\"${{5:-}}\"\nif [ \"$1\" = load ] && [ \"$2\" = image-archive ] && [ \"$3\" = --name ] && [ \"$5\" != '-' ] && [ -f \"$archive\" ]; then\n exit 0\nfi\necho \"unexpected kind invocation: $*\" >&2\nexit 1\n", + "#!/bin/sh\nset -eu\nprintf 'kind %s\\n' \"$*\" >> '{}'\narchive=\"${{5:-}}\"\nif \ + [ \"$1\" = load ] && [ \"$2\" = image-archive ] && [ \"$3\" = --name ] && [ \ + \"$5\" != '-' ] && [ -f \"$archive\" ]; then\n exit 0\nfi\necho \"unexpected \ + kind invocation: $*\" >&2\nexit 1\n", log_path.display() ), ) @@ -3062,12 +3246,19 @@ mod tests { .suffix(".tar") .tempfile_in(temp.path()) .expect("temp archive should create"); - kind_load_image_archive_with_binaries(&kind, &docker, "test-cluster", "test:image", archive) - .expect("kind image-archive fallback should use a real archive path"); + kind_load_image_archive_with_binaries( + &kind, + &docker, + "test-cluster", + "test:image", + archive, + ) + .expect("kind image-archive fallback should use a real archive path"); let log = fs::read_to_string(&log_path).expect("commands log should read"); assert!( - log.lines().any(|line| line.contains("docker image save --output ")), + log.lines() + .any(|line| line.contains("docker image save --output ")), "docker should save to a concrete archive path:\n{log}" ); assert!( @@ -3078,4 +3269,355 @@ mod tests { "kind should receive the archive filename, not stdin:\n{log}" ); } + + #[cfg(unix)] + #[test] + fn kind_load_image_archive_saves_the_requested_image_tag() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let docker = temp.path().join("docker"); + let kind = temp.path().join("kind"); + let log_path = temp.path().join("commands.log"); + + fs::write( + &docker, + format!( + r#"#!/bin/sh +set -eu +printf 'docker %s +' "$*" >> '{}' +if [ "$1" = image ] && [ "$2" = save ] && [ "$3" = --output ] && [ "$5" = 'test:image' ]; then + printf 'archive' > "$4" + exit 0 +fi +echo "unexpected docker invocation: $*" >&2 +exit 1 +"#, + log_path.display() + ), + ) + .expect("docker stub should write"); + fs::write( + &kind, + format!( + r#"#!/bin/sh +set -eu +printf 'kind %s +' "$*" >> '{}' +archive="${{5:-}}" +if [ "$1" = load ] && [ "$2" = image-archive ] && [ "$3" = --name ] && [ -f "$archive" ]; then + exit 0 +fi +echo "unexpected kind invocation: $*" >&2 +exit 1 +"#, + log_path.display() + ), + ) + .expect("kind stub should write"); + for path in [&docker, &kind] { + let mut permissions = fs::metadata(path) + .expect("stub metadata should read") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions).expect("stub should chmod"); + } + + let archive = tempfile::Builder::new() + .prefix("kind-archive-") + .suffix(".tar") + .tempfile_in(temp.path()) + .expect("temp archive should create"); + kind_load_image_archive_with_binaries( + &kind, + &docker, + "test-cluster", + "test:image", + archive, + ) + .expect("kind image-archive fallback should save the requested image tag"); + + let log = fs::read_to_string(&log_path).expect("commands log should read"); + assert!( + log.lines().any(|line| { + line.contains("docker image save --output ") && line.ends_with(" test:image") + }), + "docker save should export the requested image tag: +{log}" + ); + } + + #[cfg(unix)] + #[test] + fn load_kind_image_retries_direct_load_before_single_archive_fallback() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let docker = temp.path().join("docker"); + let kind = temp.path().join("kind"); + let log_path = temp.path().join("commands.log"); + + fs::write( + &docker, + format!( + r#"#!/bin/sh +set -eu +printf 'docker %s +' "$*" >> '{}' +if [ "$1" = image ] && [ "$2" = save ] && [ "$3" = --output ] && [ "$5" = 'test:image' ]; then + printf 'archive' > "$4" + exit 0 +fi +echo "unexpected docker invocation: $*" >&2 +exit 1 +"#, + log_path.display() + ), + ) + .expect("docker stub should write"); + fs::write( + &kind, + format!( + r#"#!/bin/sh +set -eu +printf 'kind %s +' "$*" >> '{}' +archive="${{5:-}}" +if [ "$1" = load ] && [ "$2" = docker-image ]; then + exit 1 +fi +if [ "$1" = load ] && [ "$2" = image-archive ] && [ "$3" = --name ] && [ -f "$archive" ]; then + exit 0 +fi +echo "unexpected kind invocation: $*" >&2 +exit 1 +"#, + log_path.display() + ), + ) + .expect("kind stub should write"); + for path in [&docker, &kind] { + let mut permissions = fs::metadata(path) + .expect("stub metadata should read") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions).expect("stub should chmod"); + } + + load_kind_image_with_binaries(&kind, &docker, "test-cluster", "test:image") + .expect("archive fallback should run once after direct load retries are exhausted"); + + let log = fs::read_to_string(&log_path).expect("commands log should read"); + let direct_count = log + .lines() + .filter(|line| line == &"kind load docker-image --name test-cluster test:image") + .count(); + let archive_count = log + .lines() + .filter(|line| line.starts_with("kind load image-archive --name test-cluster ")) + .count(); + let save_count = log + .lines() + .filter(|line| { + line.contains("docker image save --output ") && line.ends_with(" test:image") + }) + .count(); + assert_eq!( + direct_count, 3, + "kind load docker-image should retry exactly three times before falling back: +{log}" + ); + assert_eq!( + archive_count, 1, + "kind image-archive fallback should run once after retries are exhausted: +{log}" + ); + assert_eq!( + save_count, 1, + "docker image save should run once for the single archive fallback: +{log}" + ); + } + + #[cfg(unix)] + #[test] + fn ensure_local_image_pulls_expected_platform_when_missing() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let docker = temp.path().join("docker"); + let log_path = temp.path().join("commands.log"); + let state_path = temp.path().join("inspect-state"); + + fs::write( + &docker, + format!( + "#!/bin/sh\nset -eu\nlog_path='{}'\nstate_path='{}'\nprintf 'docker %s\\n' \"$*\" \ + >> \"$log_path\"\nif [ \"$1\" = image ] && [ \"$2\" = inspect ] && [ \"$3\" = -f \ + ] && [ \"$4\" = '{{{{.Id}}}}|{{{{.Architecture}}}}' ] && [ \"$5\" = \ + 'python:3.13-alpine' ]; then\n if [ ! -f \"$state_path\" ]; then\n exit 1\n \ + fi\n printf 'sha256:test|arm64\\n'\n exit 0\nfi\nif [ \"$1\" = pull ] && [ \ + \"$2\" = --platform ] && [ \"$3\" = 'linux/arm64' ] && [ \"$4\" = \ + 'python:3.13-alpine' ]; then\n printf pulled > \"$state_path\"\n exit \ + 0\nfi\necho \"unexpected docker invocation: $*\" >&2\nexit 1\n", + log_path.display(), + state_path.display(), + ), + ) + .expect("docker stub should write"); + let mut permissions = fs::metadata(&docker) + .expect("stub metadata should read") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&docker, permissions).expect("stub should chmod"); + + let meta = ensure_local_image_with_binaries(&docker, "python:3.13-alpine", "arm64") + .expect("missing image should be pulled for the expected platform"); + assert_eq!( + meta, + DockerImageMeta { + id: "sha256:test".to_string(), + arch: "arm64".to_string(), + } + ); + + let log = fs::read_to_string(&log_path).expect("commands log should read"); + assert!( + log.lines() + .any(|line| line == "docker pull --platform linux/arm64 python:3.13-alpine"), + "docker pull should request the expected platform:\n{log}" + ); + } + + #[cfg(unix)] + #[test] + fn ensure_local_image_repulls_expected_platform_when_local_arch_is_wrong() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let docker = temp.path().join("docker"); + let log_path = temp.path().join("commands.log"); + let state_path = temp.path().join("inspect-state"); + + fs::write( + &docker, + format!( + r#"#!/bin/sh +set -eu +log_path='{}' +state_path='{}' +printf 'docker %s +' "$*" >> "$log_path" +if [ "$1" = image ] && [ "$2" = inspect ] && [ "$3" = -f ] && [ "$4" = '{{{{.Id}}}}|{{{{.Architecture}}}}' ] && [ "$5" = 'python:3.13-alpine' ]; then + if [ -f "$state_path" ]; then + printf 'sha256:test|arm64 +' + else + printf 'sha256:stale|amd64 +' + fi + exit 0 +fi +if [ "$1" = pull ] && [ "$2" = --platform ] && [ "$3" = 'linux/arm64' ] && [ "$4" = 'python:3.13-alpine' ]; then + printf pulled > "$state_path" + exit 0 +fi +echo "unexpected docker invocation: $*" >&2 +exit 1 +"#, + log_path.display(), + state_path.display(), + ), + ) + .expect("docker stub should write"); + let mut permissions = fs::metadata(&docker) + .expect("stub metadata should read") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&docker, permissions).expect("stub should chmod"); + + let meta = ensure_local_image_with_binaries(&docker, "python:3.13-alpine", "arm64") + .expect("wrong-arch image should be repulled for the expected platform"); + assert_eq!( + meta, + DockerImageMeta { + id: "sha256:test".to_string(), + arch: "arm64".to_string(), + } + ); + + let log = fs::read_to_string(&log_path).expect("commands log should read"); + assert!( + log.lines() + .any(|line| line == "docker pull --platform linux/arm64 python:3.13-alpine"), + "wrong-arch local images should be repulled for the expected platform: +{log}" + ); + } + + #[cfg(unix)] + #[test] + fn test_app_image_builds_a_local_wrapper_image() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let docker = temp.path().join("docker"); + let log_path = temp.path().join("commands.log"); + let state_path = temp.path().join("inspect-state"); + let dockerfile_copy = temp.path().join("Dockerfile"); + + fs::write( + &docker, + format!( + r#"#!/bin/sh +set -eu +log_path='{}' +state_path='{}' +dockerfile_copy='{}' +printf 'docker %s +' "$*" >> "$log_path" +if [ "$1" = image ] && [ "$2" = inspect ] && [ "$3" = -f ] && [ "$4" = '{{{{.Id}}}}|{{{{.Architecture}}}}' ] && [ "$5" = 'python:3.13-alpine' ]; then + if [ ! -f "$state_path" ]; then + exit 1 + fi + printf 'sha256:test|arm64 +' + exit 0 +fi +if [ "$1" = pull ] && [ "$2" = --platform ] && [ "$3" = 'linux/arm64' ] && [ "$4" = 'python:3.13-alpine' ]; then + printf pulled > "$state_path" + exit 0 +fi +if [ "$1" = build ] && [ "$2" = --platform ] && [ "$3" = 'linux/arm64' ] && [ "$4" = -t ] && [ "$5" = 'amber-mixed-run-test-app:arm64' ]; then + cp "$6/Dockerfile" "$dockerfile_copy" + exit 0 +fi +echo "unexpected docker invocation: $*" >&2 +exit 1 +"#, + log_path.display(), + state_path.display(), + dockerfile_copy.display(), + ), + ) + .expect("docker stub should write"); + let mut permissions = fs::metadata(&docker) + .expect("stub metadata should read") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&docker, permissions).expect("stub should chmod"); + + let local_tag = build_test_app_image_with_binaries(&docker, "python:3.13-alpine", "arm64") + .expect("test app image should build a local wrapper image"); + assert_eq!(local_tag, "amber-mixed-run-test-app:arm64"); + + let log = fs::read_to_string(&log_path).expect("commands log should read"); + assert!( + log.lines().any(|line| { + line == "docker build --platform linux/arm64 -t amber-mixed-run-test-app:arm64" + || line.starts_with( + "docker build --platform linux/arm64 -t amber-mixed-run-test-app:arm64 ", + ) + }), + "test app image should be built as a local wrapper image: +{log}" + ); + assert_eq!( + fs::read_to_string(&dockerfile_copy).expect("copied Dockerfile should read"), + "FROM python:3.13-alpine +", + "test app image wrapper should inherit from the upstream python base image", + ); + } } diff --git a/runtime/site-controller/src/default_runtime.rs b/runtime/site-controller/src/default_runtime.rs index 2811961e..33540f41 100644 --- a/runtime/site-controller/src/default_runtime.rs +++ b/runtime/site-controller/src/default_runtime.rs @@ -350,8 +350,25 @@ fn write_vm_runtime_state(plan_root: &Path, state: &VmRuntimeState) -> Result<() const TCG_VM_STARTUP_TIMEOUT: Duration = Duration::from_secs(720); -fn vm_endpoint_forward_ready_timeout() -> Duration { - if cfg!(target_os = "macos") && env::var_os("AMBER_VM_FORCE_TCG").is_some() { +pub fn vm_uses_tcg_accel() -> bool { + #[cfg(target_os = "macos")] + { + env::var_os("AMBER_VM_FORCE_TCG").is_some() + } + + #[cfg(target_os = "linux")] + { + return env::var_os("AMBER_VM_FORCE_TCG").is_some() || !Path::new("/dev/kvm").exists(); + } + + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + env::var_os("AMBER_VM_FORCE_TCG").is_some() + } +} + +pub fn vm_endpoint_forward_ready_timeout() -> Duration { + if vm_uses_tcg_accel() { TCG_VM_STARTUP_TIMEOUT } else { Duration::from_secs(120) @@ -1822,8 +1839,12 @@ fn terminate_detached_runtime(root_pid: u32, timeout: Duration) -> Result<()> { #[cfg(test)] mod tests { + use std::sync::Mutex; + use super::*; + static VM_ACCEL_ENV_LOCK: OnceLock> = OnceLock::new(); + #[test] fn kubernetes_router_ready_prefers_local_targets_over_manager_state() { let plan = SiteControllerRuntimePlan { @@ -1942,4 +1963,25 @@ mod tests { when no embedded local router target is available", ); } + + #[test] + fn vm_endpoint_forward_ready_timeout_honors_forced_tcg() { + let _guard = VM_ACCEL_ENV_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("env lock should not be poisoned"); + let previous = env::var_os("AMBER_VM_FORCE_TCG"); + unsafe { + env::set_var("AMBER_VM_FORCE_TCG", "1"); + } + assert_eq!( + vm_endpoint_forward_ready_timeout(), + TCG_VM_STARTUP_TIMEOUT, + "forced TCG must extend VM readiness timeouts on every supported host platform", + ); + match previous { + Some(value) => unsafe { env::set_var("AMBER_VM_FORCE_TCG", value) }, + None => unsafe { env::remove_var("AMBER_VM_FORCE_TCG") }, + } + } } diff --git a/runtime/site-controller/src/default_runtime/child_runtime.rs b/runtime/site-controller/src/default_runtime/child_runtime.rs index 3adac966..66aa9a11 100644 --- a/runtime/site-controller/src/default_runtime/child_runtime.rs +++ b/runtime/site-controller/src/default_runtime/child_runtime.rs @@ -766,24 +766,29 @@ pub(super) async fn site_controller_runtime_publish_child( fs::create_dir_all(&storage_root) .into_diagnostic() .wrap_err_with(|| format!("failed to create {}", storage_root.display()))?; - let process = spawn_detached_child(&child_root, &child_root.join("site.log"), |cmd| { - cmd.arg("run-direct-init") - .arg("--plan") - .arg(Path::new(&child.artifact_root).join("direct-plan.json")) - .arg("--storage-root") - .arg(&storage_root) - .arg("--runtime-root") - .arg(&runtime_root) - .arg("--existing-peer-ports") - .arg(site_controller_runtime_child_peer_ports_path( - &app.plan, child_id, - )) - .arg("--existing-peer-identities") - .arg(site_controller_runtime_child_peer_identities_path( - &app.plan, child_id, - )) - .arg("--skip-router"); - })?; + let process = spawn_detached_child( + &child_root, + &child_root.join("site.log"), + &app.plan.launch_env, + |cmd| { + cmd.arg("run-direct-init") + .arg("--plan") + .arg(Path::new(&child.artifact_root).join("direct-plan.json")) + .arg("--storage-root") + .arg(&storage_root) + .arg("--runtime-root") + .arg(&runtime_root) + .arg("--existing-peer-ports") + .arg(site_controller_runtime_child_peer_ports_path( + &app.plan, child_id, + )) + .arg("--existing-peer-identities") + .arg(site_controller_runtime_child_peer_identities_path( + &app.plan, child_id, + )) + .arg("--skip-router"); + }, + )?; { let mut state = app.state.lock().await; let record = state @@ -866,24 +871,29 @@ pub(super) async fn site_controller_runtime_publish_child( fs::create_dir_all(&storage_root) .into_diagnostic() .wrap_err_with(|| format!("failed to create {}", storage_root.display()))?; - let process = spawn_detached_child(&child_root, &child_root.join("site.log"), |cmd| { - cmd.arg("run-vm-init") - .arg("--plan") - .arg(Path::new(&child.artifact_root).join("vm-plan.json")) - .arg("--storage-root") - .arg(&storage_root) - .arg("--runtime-root") - .arg(&runtime_root) - .arg("--existing-peer-ports") - .arg(site_controller_runtime_child_peer_ports_path( - &app.plan, child_id, - )) - .arg("--existing-peer-identities") - .arg(site_controller_runtime_child_peer_identities_path( - &app.plan, child_id, - )) - .arg("--skip-router"); - })?; + let process = spawn_detached_child( + &child_root, + &child_root.join("site.log"), + &app.plan.launch_env, + |cmd| { + cmd.arg("run-vm-init") + .arg("--plan") + .arg(Path::new(&child.artifact_root).join("vm-plan.json")) + .arg("--storage-root") + .arg(&storage_root) + .arg("--runtime-root") + .arg(&runtime_root) + .arg("--existing-peer-ports") + .arg(site_controller_runtime_child_peer_ports_path( + &app.plan, child_id, + )) + .arg("--existing-peer-identities") + .arg(site_controller_runtime_child_peer_identities_path( + &app.plan, child_id, + )) + .arg("--skip-router"); + }, + )?; { let mut state = app.state.lock().await; let record = state diff --git a/runtime/site-controller/src/default_runtime/site_runtime_support.rs b/runtime/site-controller/src/default_runtime/site_runtime_support.rs index c616c400..df3d6d8a 100644 --- a/runtime/site-controller/src/default_runtime/site_runtime_support.rs +++ b/runtime/site-controller/src/default_runtime/site_runtime_support.rs @@ -738,12 +738,23 @@ pub(super) fn amber_cli_executable() -> Result { pub(crate) fn spawn_detached_child( work_dir: &Path, log_path: &Path, + extra_env: &BTreeMap, + build: impl FnOnce(&mut Command), +) -> Result { + let exe = amber_cli_executable()?; + spawn_detached_child_with_executable(&exe, work_dir, log_path, extra_env, build) +} + +fn spawn_detached_child_with_executable( + executable: &Path, + work_dir: &Path, + log_path: &Path, + extra_env: &BTreeMap, build: impl FnOnce(&mut Command), ) -> Result { #[cfg(unix)] use std::os::unix::process::CommandExt as _; - let exe = amber_cli_executable()?; let log = fs::File::create(log_path) .into_diagnostic() .wrap_err_with(|| format!("failed to create log {}", log_path.display()))?; @@ -751,8 +762,9 @@ pub(crate) fn spawn_detached_child( .try_clone() .into_diagnostic() .wrap_err("failed to clone log handle")?; - let mut cmd = Command::new(exe); + let mut cmd = Command::new(executable); cmd.current_dir(work_dir); + cmd.envs(extra_env); cmd.stdin(Stdio::null()); cmd.stdout(Stdio::from(log)); cmd.stderr(Stdio::from(log_err)); @@ -1245,6 +1257,49 @@ mod tests { assert_eq!(tree, vec![7, 9, 8, 42]); } + #[cfg(unix)] + #[test] + fn spawn_detached_child_propagates_launch_env() { + use std::os::unix::fs::PermissionsExt as _; + + let temp = tempdir().expect("temp dir should create"); + let executable = temp.path().join("amber-stub"); + let log_path = temp.path().join("child.log"); + let env_output_path = temp.path().join("env.txt"); + fs::write( + &executable, + r#"#!/bin/sh +set -eu +printf '%s' "${AMBER_VM_FORCE_TCG:-missing}" > "$1" +"#, + ) + .expect("stub executable should write"); + let mut permissions = fs::metadata(&executable) + .expect("stub metadata should read") + .permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&executable, permissions).expect("stub should chmod"); + + let launch_env = BTreeMap::from([("AMBER_VM_FORCE_TCG".to_string(), "1".to_string())]); + let mut child = super::spawn_detached_child_with_executable( + &executable, + temp.path(), + &log_path, + &launch_env, + |cmd| { + cmd.arg(&env_output_path); + }, + ) + .expect("detached child should spawn"); + let status = child.wait().expect("stub child should exit"); + assert!(status.success(), "stub child should succeed: {status}"); + assert_eq!( + fs::read_to_string(&env_output_path).expect("env output should read"), + "1", + "detached children must inherit launch_env so VM child runtimes see TCG forcing", + ); + } + #[test] fn in_cluster_kubernetes_controller_skips_namespace_bootstrap() { let plan = kubernetes_supervisor_plan(None); diff --git a/runtime/site-controller/src/lib.rs b/runtime/site-controller/src/lib.rs index cdcf5430..da69fb41 100644 --- a/runtime/site-controller/src/lib.rs +++ b/runtime/site-controller/src/lib.rs @@ -84,7 +84,8 @@ pub use self::{ inject_site_controller_peer_router_routes, observability_endpoint_for_site, prepare_kubernetes_artifact_namespace, reserve_loopback_port, router_mesh_addr_for_consumer, set_compose_router_published_mesh_port, - set_site_artifact_mesh_identity_seed, site_controller_peer_router_url, walk_files, + set_site_artifact_mesh_identity_seed, site_controller_peer_router_url, + vm_endpoint_forward_ready_timeout, vm_uses_tcg_accel, walk_files, }, runtime_api::{ DesiredExportPeerOverlay, DesiredExternalSlotOverlay, DesiredRouteOverlay, LaunchedSite, From 16f5244962162d9cfe79184672111d3abaeb93df Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Fri, 17 Apr 2026 10:50:38 +0800 Subject: [PATCH 09/25] fix tests --- runtime/site-controller/src/default_runtime.rs | 2 +- runtime/site-controller/src/tests.rs | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/runtime/site-controller/src/default_runtime.rs b/runtime/site-controller/src/default_runtime.rs index 33540f41..4e27e671 100644 --- a/runtime/site-controller/src/default_runtime.rs +++ b/runtime/site-controller/src/default_runtime.rs @@ -358,7 +358,7 @@ pub fn vm_uses_tcg_accel() -> bool { #[cfg(target_os = "linux")] { - return env::var_os("AMBER_VM_FORCE_TCG").is_some() || !Path::new("/dev/kvm").exists(); + env::var_os("AMBER_VM_FORCE_TCG").is_some() || !Path::new("/dev/kvm").exists() } #[cfg(not(any(target_os = "macos", target_os = "linux")))] diff --git a/runtime/site-controller/src/tests.rs b/runtime/site-controller/src/tests.rs index 6191e79b..18547e24 100644 --- a/runtime/site-controller/src/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -1,5 +1,3 @@ -#[cfg(all(unix, not(target_os = "macos")))] -use std::os::unix::fs::MetadataExt; use std::{ fs, future::Future, From 69c3f9982d8384fab0314f4421f261e05ba212f6 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Sat, 18 Apr 2026 20:49:39 +0800 Subject: [PATCH 10/25] Decentralize traffic --- cli/src/direct_runtime/preview.rs | 47 +- cli/src/direct_runtime/runtime.rs | 35 - cli/src/mixed_run.rs | 26 +- cli/src/mixed_run/launch_bundle.rs | 412 ++--- cli/src/mixed_run/supervisor.rs | 43 +- cli/src/mixed_run/tests.rs | 38 +- cli/src/run_inputs.rs | 2 + cli/src/tests.rs | 37 - cli/src/vm_runtime/preview.rs | 31 +- cli/src/vm_runtime/tests.rs | 108 +- cli/tests/mixed_run/smoke.rs | 20 +- cli/tests/mixed_run/support.rs | 3 - compiler/src/reporter/mod.rs | 25 + compiler/src/run_plan.rs | 1417 ++++++++++++++++- compiler/src/targets/direct/mod.rs | 71 +- compiler/src/targets/mesh/addressing.rs | 29 +- .../src/targets/mesh/docker_compose/mod.rs | 29 +- .../targets/mesh/docker_compose/tests/mod.rs | 68 + compiler/src/targets/mesh/kubernetes/mod.rs | 46 +- .../src/targets/mesh/kubernetes/tests/mod.rs | 73 + .../mesh/kubernetes/tests/rendering.rs | 8 +- compiler/src/targets/mesh/mesh_config.rs | 281 +++- compiler/src/targets/mesh/plan.rs | 10 +- compiler/src/targets/mesh/ports.rs | 16 + compiler/src/targets/vm/mod.rs | 97 ++ runtime/mesh/src/dynamic_caps.rs | 5 +- runtime/mesh/src/lib.rs | 5 +- runtime/router/src/control.rs | 12 +- runtime/router/src/dynamic_caps.rs | 135 +- runtime/router/src/external.rs | 5 +- runtime/router/src/lib.rs | 23 - runtime/router/src/tests.rs | 225 ++- runtime/site-controller/src/api.rs | 31 +- .../site-controller/src/control_state_mcp.rs | 2 +- .../site-controller/src/default_runtime.rs | 6 - .../src/default_runtime/compose_controller.rs | 240 ++- .../src/default_runtime/dynamic_routes.rs | 68 +- .../default_runtime/kubernetes_controller.rs | 422 +++-- .../src/default_runtime/site_artifacts.rs | 56 +- .../default_runtime/site_runtime_support.rs | 5 - runtime/site-controller/src/http.rs | 10 +- runtime/site-controller/src/lib.rs | 4 +- runtime/site-controller/src/orchestration.rs | 181 +-- runtime/site-controller/src/planner.rs | 184 ++- runtime/site-controller/src/runtime_api.rs | 2 - .../site-controller/src/site_controller.rs | 99 +- runtime/site-controller/src/state.rs | 81 +- runtime/site-controller/src/tests.rs | 759 +++++---- 48 files changed, 4005 insertions(+), 1527 deletions(-) diff --git a/cli/src/direct_runtime/preview.rs b/cli/src/direct_runtime/preview.rs index d3c89dea..dce44c03 100644 --- a/cli/src/direct_runtime/preview.rs +++ b/cli/src/direct_runtime/preview.rs @@ -105,9 +105,47 @@ pub(crate) fn component_program_spec( network: ProcessNetwork::Host, }) } + DirectProgramExecutionPlan::InternalSiteController => { + let (program, args) = + split_entrypoint(&internal_site_controller_entrypoint(runtime_root)?)?; + Ok(ProcessSpec { + name: component.program.log_name.clone(), + program, + args, + env: BTreeMap::new(), + work_dir, + sandbox: ProcessSandbox::Unsandboxed, + drop_all_caps: true, + #[cfg(target_os = "linux")] + read_only_mounts: Vec::new(), + writable_dirs, + bind_dirs: Vec::new(), + bind_mounts, + hidden_paths: Vec::new(), + network: ProcessNetwork::Host, + }) + } } } +fn internal_site_controller_plan_path(runtime_root: &Path) -> PathBuf { + let site_state_root = runtime_root.parent().unwrap_or(runtime_root); + mixed_run::site_controller_plan_path(site_state_root) +} + +fn internal_site_controller_entrypoint(runtime_root: &Path) -> Result> { + let command = mixed_run::site_controller_command()?; + let mut argv = vec![command.executable.display().to_string()]; + argv.extend(command.prefix_args.iter().map(|arg| (*arg).to_string())); + argv.push("--plan".to_string()); + argv.push( + internal_site_controller_plan_path(runtime_root) + .display() + .to_string(), + ); + Ok(argv) +} + pub(crate) fn build_direct_site_launch_preview( plan_path: &Path, storage_root: &Path, @@ -265,9 +303,6 @@ pub(crate) fn build_direct_site_launch_preview( ); for passthrough in &component.sidecar.env_passthrough { if let Ok(value) = env::var(passthrough) { - #[cfg(target_os = "linux")] - let value = - rewrite_sidecar_env_passthrough_for_slirp(passthrough.as_str(), value.as_str()); env.insert(passthrough.clone(), value); } } @@ -326,6 +361,7 @@ pub(crate) fn build_direct_site_launch_preview( DirectProgramExecutionPlan::HelperRunner { .. } => { Some(direct_resolved_process_preview(&spec.env)?) } + DirectProgramExecutionPlan::InternalSiteController => None, }; processes.push(direct_process_preview( spec, @@ -343,12 +379,12 @@ pub(crate) fn build_direct_site_launch_preview( }) } -fn missing_existing_peer_identity(err: &miette::Report) -> bool { +pub(crate) fn missing_existing_peer_identity(err: &miette::Report) -> bool { err.to_string() .contains("mesh provision plan requires existing peer identity") } -fn preview_placeholder_peer_mesh_state( +pub(crate) fn preview_placeholder_peer_mesh_state( mesh_plan: &MeshProvisionPlan, ) -> (BTreeMap, BTreeMap) { let target_ids = mesh_plan @@ -893,6 +929,7 @@ pub(crate) fn component_execution_program_path( } Ok(None) } + DirectProgramExecutionPlan::InternalSiteController => Ok(None), } } diff --git a/cli/src/direct_runtime/runtime.rs b/cli/src/direct_runtime/runtime.rs index f0c14c37..e5b88459 100644 --- a/cli/src/direct_runtime/runtime.rs +++ b/cli/src/direct_runtime/runtime.rs @@ -392,11 +392,6 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { ); for passthrough in &component.sidecar.env_passthrough { if let Ok(value) = env::var(passthrough) { - #[cfg(target_os = "linux")] - let value = rewrite_sidecar_env_passthrough_for_slirp( - passthrough.as_str(), - value.as_str(), - ); env.insert(passthrough.clone(), value); } } @@ -1283,36 +1278,6 @@ pub(crate) fn rewrite_peer_addr_for_slirp_gateway(peer_addr: &str) -> String { SocketAddr::from((Ipv4Addr::new(10, 0, 2, 2), addr.port())).to_string() } -#[cfg(target_os = "linux")] -pub(crate) fn rewrite_sidecar_env_passthrough_for_slirp(name: &str, value: &str) -> String { - if name != amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV { - return value.to_string(); - } - rewrite_loopback_url_for_slirp_gateway(value) -} - -#[cfg(target_os = "linux")] -pub(crate) fn rewrite_loopback_url_for_slirp_gateway(value: &str) -> String { - let Ok(mut url) = Url::parse(value) else { - return value.to_string(); - }; - let Some(host) = url.host_str() else { - return value.to_string(); - }; - let is_loopback = host.eq_ignore_ascii_case("localhost") - || host - .parse::() - .map(|addr| addr.is_loopback()) - .unwrap_or(false); - if !is_loopback { - return value.to_string(); - } - if url.set_host(Some("10.0.2.2")).is_err() { - return value.to_string(); - } - url.to_string() -} - pub(crate) fn read_mesh_config_public(path: &Path) -> Result { let raw = fs::read_to_string(path) .map_err(|err| miette::miette!("failed to read mesh config {}: {err}", path.display()))?; diff --git a/cli/src/mixed_run.rs b/cli/src/mixed_run.rs index 87b7e789..d404a3ca 100644 --- a/cli/src/mixed_run.rs +++ b/cli/src/mixed_run.rs @@ -365,8 +365,6 @@ pub(crate) struct SiteReceipt { #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) router_public_key_b64: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub(crate) site_controller_pid: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) site_controller_url: Option, } @@ -413,8 +411,6 @@ struct SiteManagerState { #[serde(default, skip_serializing_if = "Option::is_none")] router_public_key_b64: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - site_controller_pid: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] site_controller_url: Option, #[serde(default, skip_serializing_if = "Option::is_none")] last_error: Option, @@ -451,8 +447,6 @@ pub(crate) struct SiteSupervisorPlan { #[serde(default, skip_serializing_if = "Option::is_none")] observability_endpoint: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - site_controller_plan_path: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] site_controller_url: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] launch_env: BTreeMap, @@ -467,7 +461,6 @@ struct SupervisorPlanInput<'a> { artifact_dir: &'a Path, site_state_root: &'a Path, observability_endpoint: Option<&'a str>, - site_controller_plan_path: Option<&'a Path>, site_controller_url: Option<&'a str>, } @@ -477,15 +470,20 @@ struct MaterializedObservability { plan_path: Option, } +#[derive(Clone, Debug)] +pub(crate) struct MaterializedSiteController { + pub(crate) state_path: PathBuf, + pub(crate) plan_path: PathBuf, + pub(crate) listen_addr: SocketAddr, + pub(crate) url: String, +} + #[derive(Clone, Debug)] pub(crate) struct MaterializedSite { pub(crate) site_plan: RunSitePlan, pub(crate) artifact_dir: PathBuf, pub(crate) site_state_root: PathBuf, - pub(crate) controller_state_path: PathBuf, - pub(crate) controller_plan_path: PathBuf, - pub(crate) controller_listen_addr: SocketAddr, - pub(crate) controller_url: String, + pub(crate) controller: Option, pub(crate) base_supervisor_plan: SiteSupervisorPlan, } @@ -642,7 +640,6 @@ struct SupervisorRuntime { site_process: Option, site_started: bool, port_forward: Option, - site_controller: Option, last_start_attempt: Option, last_stitch_refresh: Option, ready_since: Option, @@ -685,7 +682,6 @@ fn site_state_from_receipt( kubernetes_consumer_router_mesh_addr: site.kubernetes_consumer_router_mesh_addr.clone(), router_identity_id: site.router_identity_id.clone(), router_public_key_b64: site.router_public_key_b64.clone(), - site_controller_pid: site.site_controller_pid, site_controller_url: site.site_controller_url.clone(), last_error, } @@ -825,7 +821,6 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { site_process: None, site_started: false, port_forward: None, - site_controller: None, last_start_attempt: None, last_stitch_refresh: None, ready_since: None, @@ -871,7 +866,6 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { kubernetes_consumer_router_mesh_addr, router_identity_id: None, router_public_key_b64: None, - site_controller_pid: None, site_controller_url: None, last_error: None, }, @@ -908,7 +902,6 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { kubernetes_consumer_router_mesh_addr, router_identity_id: None, router_public_key_b64: None, - site_controller_pid: None, site_controller_url: None, last_error: Some("coordinator exited before commit".to_string()), }, @@ -1024,7 +1017,6 @@ pub(crate) async fn run_site_supervisor(plan_path: PathBuf) -> Result<()> { kubernetes_consumer_router_mesh_addr, router_identity_id: Some(discovery.router_identity.id), router_public_key_b64: Some(public_key_b64), - site_controller_pid: None, site_controller_url: None, last_error: None, }, diff --git a/cli/src/mixed_run/launch_bundle.rs b/cli/src/mixed_run/launch_bundle.rs index d4ee7ace..671a3e8b 100644 --- a/cli/src/mixed_run/launch_bundle.rs +++ b/cli/src/mixed_run/launch_bundle.rs @@ -31,6 +31,54 @@ pub(super) fn site_controller_local_router_control(kind: SiteKind, artifact_dir: } } +fn site_controller_component_port(site_plan: &RunSitePlan) -> Result> { + let scenario = + amber_scenario::Scenario::try_from(site_plan.scenario_ir.clone()).map_err(|err| { + miette::miette!( + "failed to parse site scenario for router `{}`: {err}", + site_plan.router_identity_id + ) + })?; + let mut controller_ports = Vec::new(); + for (_, component) in scenario.components_iter() { + if amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .is_none() + { + continue; + } + let port = component + .program + .as_ref() + .and_then(|program| program.network()) + .and_then(|network| network.endpoints.first()) + .map(|endpoint| endpoint.port) + .ok_or_else(|| { + miette::miette!( + "synthetic framework.component controller `{}` on site `{}` is missing its \ + endpoint", + component.moniker, + site_plan.router_identity_id + ) + })?; + controller_ports.push((component.moniker.clone(), port)); + } + match controller_ports.as_slice() { + [] => Ok(None), + [(_, port)] => Ok(Some(*port)), + _ => Err(miette::miette!( + "site `{}` has multiple synthetic framework.component controllers: {}", + site_plan.router_identity_id, + controller_ports + .iter() + .map(|(moniker, _)| moniker.as_str()) + .collect::>() + .join(", ") + )), + } +} + pub(crate) fn dry_run_run_plan( source_plan_path: Option<&Path>, run_plan: &RunPlan, @@ -85,7 +133,7 @@ pub(super) fn materialize_launch_bundle( write_json(&run_plan_path, run_plan)?; let site_controller_image = site_controller_image_reference()?; - let controller_auth_token = amber_site_controller::generate_framework_auth_token( + let control_state_auth_token = amber_site_controller::generate_control_state_auth_token( &run_plan.mesh_scope, "site-controller", ); @@ -111,32 +159,33 @@ pub(super) fn materialize_launch_bundle( let artifact_dir = materialize_site_artifacts(&sites_root, site_id, site_plan)?; amber_site_controller::set_site_artifact_mesh_identity_seed(&artifact_dir, run_id)?; let site_state_root = state_root.join(site_id); - let controller_state_path = site_state_root.join("site-controller-state.json"); - let controller_plan_path = site_controller_plan_path(&site_state_root); - let (controller_listen_addr, controller_url) = match site_plan.site.kind { - SiteKind::Direct | SiteKind::Vm => { - let port = reserve_loopback_port()?; - let listen_addr = host_service_bind_addr_for_consumer(site_plan.site.kind, port); - (listen_addr, format!("http://127.0.0.1:{port}")) - } - SiteKind::Compose | SiteKind::Kubernetes => ( - SocketAddr::from(([0, 0, 0, 0], amber_site_controller::SITE_CONTROLLER_PORT)), - format!( + let controller = site_controller_component_port(site_plan)?.map(|controller_port| { + let listen_addr = match site_plan.site.kind { + SiteKind::Direct | SiteKind::Vm => { + host_service_bind_addr_for_consumer(site_plan.site.kind, controller_port) + } + SiteKind::Compose | SiteKind::Kubernetes => { + SocketAddr::from(([0, 0, 0, 0], controller_port)) + } + }; + let url = match site_plan.site.kind { + SiteKind::Direct | SiteKind::Vm => { + amber_site_controller::authority_url_for_listen_addr(listen_addr) + } + SiteKind::Compose | SiteKind::Kubernetes => format!( "http://{}:{}", amber_site_controller::SITE_CONTROLLER_SERVICE_NAME, - amber_site_controller::SITE_CONTROLLER_PORT + controller_port ), - ), - }; + }; + MaterializedSiteController { + state_path: site_state_root.join("site-controller-state.json"), + plan_path: site_controller_plan_path(&site_state_root), + listen_addr, + url, + } + }); let mut framework_env = BTreeMap::new(); - framework_env.insert( - amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV.to_string(), - controller_url.clone(), - ); - framework_env.insert( - amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV.to_string(), - controller_auth_token.clone(), - ); framework_env.insert( amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV.to_string(), dynamic_caps_token_verify_key_b64.clone(), @@ -167,12 +216,9 @@ pub(super) fn materialize_launch_bundle( artifact_dir: &artifact_dir, site_state_root: &site_state_root, observability_endpoint, - site_controller_plan_path: matches!( - site_plan.site.kind, - SiteKind::Direct | SiteKind::Vm - ) - .then_some(controller_plan_path.as_path()), - site_controller_url: Some(controller_url.as_str()), + site_controller_url: controller + .as_ref() + .map(|controller| controller.url.as_str()), }, launch_env.clone(), )?; @@ -184,15 +230,17 @@ pub(super) fn materialize_launch_bundle( router_mesh_port, )?; } - let controller_state = amber_site_controller::build_site_controller_state( - run_id, - run_plan, - site_id, - site_index, - run_plan.sites.len(), - &dynamic_capability_signing_seed_b64, - )?; - amber_site_controller::write_control_state(&controller_state_path, &controller_state)?; + if let Some(controller) = &controller { + let controller_state = amber_site_controller::build_site_controller_state( + run_id, + run_plan, + site_id, + site_index, + run_plan.sites.len(), + &dynamic_capability_signing_seed_b64, + )?; + amber_site_controller::write_control_state(&controller.state_path, &controller_state)?; + } write_json( &desired_links_path(&site_state_root), &DesiredLinkState { @@ -210,10 +258,7 @@ pub(super) fn materialize_launch_bundle( site_plan: site_plan.clone(), artifact_dir, site_state_root, - controller_state_path, - controller_plan_path, - controller_listen_addr, - controller_url, + controller, base_supervisor_plan, }, ); @@ -228,144 +273,153 @@ pub(super) fn materialize_launch_bundle( )) }) .collect::>>()?; + let controller_sites = sites + .iter() + .filter(|(_, site)| site.controller.is_some()) + .map(|(site_id, site)| { + ( + site_id.clone(), + site.site_plan.router_identity_id.clone(), + site.site_plan.site.kind, + ) + }) + .collect::>(); for (site_id, site) in &mut sites { - let allowed_issuers = run_plan - .sites - .iter() - .filter(|(peer_site_id, _)| *peer_site_id != site_id) - .map(|(_, peer_site)| peer_site.router_identity_id.clone()) - .collect::>(); - let route_listen_addr = - site_controller_route_listen_addr(site.site_plan.site.kind).to_string(); - let mut peer_site_router_urls = BTreeMap::new(); - let mut peer_router_identities = BTreeMap::new(); - let mut peer_router_mesh_addrs = BTreeMap::new(); - let mut controller_routes = Vec::new(); - for (peer_site_id, peer_site_plan) in &run_plan.sites { - if peer_site_id == site_id { - continue; - } - let route_port = reserve_loopback_port()?; - peer_site_router_urls.insert( - peer_site_id.clone(), - amber_site_controller::site_controller_peer_router_url( + if let Some(controller) = &site.controller { + let allowed_issuers = controller_sites + .iter() + .filter(|(peer_site_id, _, _)| peer_site_id != site_id) + .map(|(_, router_identity_id, _)| router_identity_id.clone()) + .collect::>(); + let route_listen_addr = + site_controller_route_listen_addr(site.site_plan.site.kind).to_string(); + let mut peer_site_router_urls = BTreeMap::new(); + let mut peer_router_identities = BTreeMap::new(); + let mut peer_router_mesh_addrs = BTreeMap::new(); + let mut controller_routes = Vec::new(); + for (peer_site_id, peer_router_identity_id, peer_site_kind) in &controller_sites { + if peer_site_id == site_id { + continue; + } + let route_port = reserve_loopback_port()?; + peer_site_router_urls.insert( + peer_site_id.clone(), + amber_site_controller::site_controller_peer_router_url( + site.site_plan.site.kind, + route_port, + ), + ); + peer_router_identities.insert( + peer_site_id.clone(), + planned_router_identity(run_id, &run_plan.mesh_scope, peer_router_identity_id), + ); + let peer_router_mesh_addr = planned_router_mesh_addrs + .get(peer_site_id) + .expect("planned router mesh addr should exist for every site"); + peer_router_mesh_addrs.insert(peer_site_id.clone(), peer_router_mesh_addr.clone()); + let peer_addr = amber_site_controller::router_mesh_addr_for_consumer( + *peer_site_kind, site.site_plan.site.kind, - route_port, - ), - ); - peer_router_identities.insert( - peer_site_id.clone(), - planned_router_identity( - run_id, - &run_plan.mesh_scope, - &peer_site_plan.router_identity_id, - ), - ); - let peer_router_mesh_addr = planned_router_mesh_addrs - .get(peer_site_id) - .expect("planned router mesh addr should exist for every site"); - peer_router_mesh_addrs.insert(peer_site_id.clone(), peer_router_mesh_addr.clone()); - let peer_addr = amber_site_controller::router_mesh_addr_for_consumer( - peer_site_plan.site.kind, - site.site_plan.site.kind, - peer_router_mesh_addr, - )?; - controller_routes.push(amber_site_controller::SiteControllerPeerRouterRoute { - site_id: peer_site_id.clone(), - peer_router: planned_router_identity( - run_id, - &run_plan.mesh_scope, - &peer_site_plan.router_identity_id, - ), - peer_addr, - listen_addr: route_listen_addr.clone(), - listen_port: route_port, - }); - } - - write_json( - &site_existing_peer_ports_path(&site.site_state_root), - &peer_router_ports_by_identity(&peer_router_identities, &peer_router_mesh_addrs)?, - )?; - write_json( - &site_existing_peer_identities_path(&site.site_state_root), - &peer_router_identities_by_id(&peer_router_identities), - )?; + peer_router_mesh_addr, + )?; + controller_routes.push(amber_site_controller::SiteControllerPeerRouterRoute { + site_id: peer_site_id.clone(), + peer_router: planned_router_identity( + run_id, + &run_plan.mesh_scope, + peer_router_identity_id, + ), + peer_addr, + listen_addr: route_listen_addr.clone(), + listen_port: route_port, + }); + } - amber_site_controller::inject_site_controller_peer_router_routes( - &site.artifact_dir, - site_id, - &allowed_issuers, - &controller_routes, - )?; - let local_router_control = - site_controller_local_router_control(site.site_plan.site.kind, &site.artifact_dir); - let published_router_mesh_addr = planned_router_mesh_addrs.get(site_id).map(String::as_str); - let compose_consumer_router_mesh_addr = published_router_mesh_addr - .map(|addr| { - amber_site_controller::router_mesh_addr_for_consumer( - site.site_plan.site.kind, - SiteKind::Compose, - addr, - ) - }) - .transpose()?; - let kubernetes_consumer_router_mesh_addr = published_router_mesh_addr - .map(|addr| { - amber_site_controller::router_mesh_addr_for_consumer( - site.site_plan.site.kind, - SiteKind::Kubernetes, - addr, - ) - }) - .transpose()?; + write_json( + &site_existing_peer_ports_path(&site.site_state_root), + &peer_router_ports_by_identity(&peer_router_identities, &peer_router_mesh_addrs)?, + )?; + write_json( + &site_existing_peer_identities_path(&site.site_state_root), + &peer_router_identities_by_id(&peer_router_identities), + )?; - amber_site_controller::write_site_controller_plan( - &site.controller_plan_path, - run_id, - &run_plan.mesh_scope, - site_id, - site.site_plan.site.kind, - site.controller_listen_addr, - &site.controller_url, - &site.site_plan.router_identity_id, - &peer_site_router_urls, - &peer_router_identities, - &peer_router_mesh_addrs, - Some(local_router_control.as_str()), - published_router_mesh_addr, - compose_consumer_router_mesh_addr.as_deref(), - kubernetes_consumer_router_mesh_addr.as_deref(), - &site.controller_state_path, - bundle_root, - &state_root, - &site.site_state_root, - &site.artifact_dir, - &controller_auth_token, - &dynamic_caps_token_verify_key_b64, - site.base_supervisor_plan.storage_root.as_deref(), - site.base_supervisor_plan.runtime_root.as_deref(), - site.base_supervisor_plan.router_mesh_port, - site.base_supervisor_plan.compose_project.as_deref(), - site.base_supervisor_plan.kubernetes_namespace.as_deref(), - site.base_supervisor_plan.context.as_deref(), - site.base_supervisor_plan.observability_endpoint.as_deref(), - &site.base_supervisor_plan.launch_env, - )?; - match site.site_plan.site.kind { - SiteKind::Compose => amber_site_controller::inject_compose_site_controller( + amber_site_controller::inject_site_controller_peer_router_routes( &site.artifact_dir, - &read_json(&site.controller_plan_path, "site controller plan")?, - &site.controller_plan_path, - site_controller_image.as_str(), - )?, - SiteKind::Kubernetes => amber_site_controller::inject_kubernetes_site_controller( + site_id, + &allowed_issuers, + &controller_routes, + )?; + let local_router_control = + site_controller_local_router_control(site.site_plan.site.kind, &site.artifact_dir); + let published_router_mesh_addr = + planned_router_mesh_addrs.get(site_id).map(String::as_str); + let compose_consumer_router_mesh_addr = published_router_mesh_addr + .map(|addr| { + amber_site_controller::router_mesh_addr_for_consumer( + site.site_plan.site.kind, + SiteKind::Compose, + addr, + ) + }) + .transpose()?; + let kubernetes_consumer_router_mesh_addr = published_router_mesh_addr + .map(|addr| { + amber_site_controller::router_mesh_addr_for_consumer( + site.site_plan.site.kind, + SiteKind::Kubernetes, + addr, + ) + }) + .transpose()?; + + amber_site_controller::write_site_controller_plan( + &controller.plan_path, + run_id, + &run_plan.mesh_scope, + site_id, + site.site_plan.site.kind, + controller.listen_addr, + &controller.url, + &site.site_plan.router_identity_id, + &peer_site_router_urls, + &peer_router_identities, + &peer_router_mesh_addrs, + Some(local_router_control.as_str()), + published_router_mesh_addr, + compose_consumer_router_mesh_addr.as_deref(), + kubernetes_consumer_router_mesh_addr.as_deref(), + &controller.state_path, + bundle_root, + &state_root, + &site.site_state_root, &site.artifact_dir, - &read_json(&site.controller_plan_path, "site controller plan")?, - site_controller_image.as_str(), - )?, - SiteKind::Direct | SiteKind::Vm => {} + &control_state_auth_token, + &dynamic_caps_token_verify_key_b64, + site.base_supervisor_plan.storage_root.as_deref(), + site.base_supervisor_plan.runtime_root.as_deref(), + site.base_supervisor_plan.router_mesh_port, + site.base_supervisor_plan.compose_project.as_deref(), + site.base_supervisor_plan.kubernetes_namespace.as_deref(), + site.base_supervisor_plan.context.as_deref(), + site.base_supervisor_plan.observability_endpoint.as_deref(), + &site.base_supervisor_plan.launch_env, + )?; + match site.site_plan.site.kind { + SiteKind::Compose => amber_site_controller::inject_compose_site_controller( + &site.artifact_dir, + &read_json(&controller.plan_path, "site controller plan")?, + &controller.plan_path, + site_controller_image.as_str(), + )?, + SiteKind::Kubernetes => amber_site_controller::inject_kubernetes_site_controller( + &site.artifact_dir, + &read_json(&controller.plan_path, "site controller plan")?, + site_controller_image.as_str(), + )?, + SiteKind::Direct | SiteKind::Vm => {} + } } write_json( &site_supervisor_plan_path(&site.site_state_root), @@ -710,17 +764,6 @@ pub(super) fn preview_external_slot_url( pub(super) fn site_launch_commands(plan: &SiteSupervisorPlan) -> Result> { let exe = super::amber_cli_executable()?; let mut commands = Vec::new(); - if let Some(plan_path) = plan.site_controller_plan_path.as_deref() { - let controller = super::site_controller_command()?; - let mut argv = vec![controller.executable.display().to_string()]; - argv.extend(controller.prefix_args.iter().map(|arg| (*arg).to_string())); - argv.extend(["--plan".to_string(), plan_path.to_string()]); - commands.push(LaunchCommandPreview { - argv, - env: plan.launch_env.clone(), - current_dir: Some(plan.site_state_root.clone()), - }); - } let site_commands = match plan.kind { SiteKind::Direct => { let mut argv = vec![ @@ -1231,7 +1274,6 @@ pub(crate) async fn run_run_plan_with_id( .clone(), router_identity_id: receipt.router_identity_id.clone(), router_public_key_b64: receipt.router_public_key_b64.clone(), - site_controller_pid: receipt.site_controller_pid, site_controller_url: receipt.site_controller_url.clone(), last_error: Some("coordinator cleanup after failed startup".to_string()), }, diff --git a/cli/src/mixed_run/supervisor.rs b/cli/src/mixed_run/supervisor.rs index e7728719..ac5200ab 100644 --- a/cli/src/mixed_run/supervisor.rs +++ b/cli/src/mixed_run/supervisor.rs @@ -265,9 +265,6 @@ pub(super) fn build_supervisor_plan( None }, observability_endpoint: input.observability_endpoint.map(ToOwned::to_owned), - site_controller_plan_path: input - .site_controller_plan_path - .map(|path| path.display().to_string()), site_controller_url: input.site_controller_url.map(ToOwned::to_owned), launch_env, }) @@ -584,7 +581,6 @@ pub(super) fn launched_site_from_state( .clone(), router_identity_id: state.router_identity_id.clone(), router_public_key_b64: state.router_public_key_b64.clone(), - site_controller_pid: state.site_controller_pid, site_controller_url: state.site_controller_url.clone(), }, router_control, @@ -637,26 +633,6 @@ pub(super) async fn ensure_site_running( ) -> Result<()> { reap_child(&mut runtime.site_process)?; reap_child(&mut runtime.port_forward)?; - reap_child(&mut runtime.site_controller)?; - - if matches!(plan.kind, SiteKind::Direct | SiteKind::Vm) - && runtime.site_controller.is_none() - && let Some(plan_path) = plan.site_controller_plan_path.as_deref() - { - let controller = super::site_controller_command()?; - runtime.site_controller = Some(spawn_runtime_process_with_executable( - &controller.executable, - &PathBuf::from(&plan.site_state_root), - "site-controller.log", - &plan.launch_env, - |cmd| { - for arg in &controller.prefix_args { - cmd.arg(arg); - } - cmd.arg("--plan").arg(plan_path); - }, - )?); - } match plan.kind { SiteKind::Direct => { @@ -808,6 +784,7 @@ pub(super) async fn try_discover_site( }?; if discovery.is_none() && plan.kind == SiteKind::Compose + && compose_site_requires_controller(plan) && !compose_site_controller_started(plan)? { runtime.site_started = false; @@ -1000,8 +977,12 @@ pub(super) fn local_site_controller_addr(plan: &SiteSupervisorPlan) -> Result bool { + plan.kind == SiteKind::Compose && plan.site_controller_url.is_some() +} + pub(super) fn compose_site_controller_container_name(plan: &SiteSupervisorPlan) -> Option { - (plan.kind == SiteKind::Compose) + compose_site_requires_controller(plan) .then_some(plan.compose_project.as_deref()?) .map(|project| { format!( @@ -1193,7 +1174,7 @@ pub(super) async fn try_discover_compose_site( else { return Ok(None); }; - if !compose_site_controller_ready(plan)? { + if compose_site_requires_controller(plan) && !compose_site_controller_ready(plan)? { return Ok(None); } Ok(Some(discovery)) @@ -1326,21 +1307,15 @@ pub(super) async fn cleanup_site( ) -> Result<()> { reap_child(&mut runtime.site_process)?; reap_child(&mut runtime.port_forward)?; - reap_child(&mut runtime.site_controller)?; - if let Some(child) = runtime.site_process.as_mut() { stop_child(child).await?; } if let Some(child) = runtime.port_forward.as_mut() { stop_child(child).await?; } - if let Some(child) = runtime.site_controller.as_mut() { - stop_child(child).await?; - } runtime.site_process = None; runtime.site_started = false; runtime.port_forward = None; - runtime.site_controller = None; match plan.kind { SiteKind::Compose => { @@ -1425,7 +1400,6 @@ pub(super) fn build_site_state( kubernetes_consumer_router_mesh_addr, router_identity_id, router_public_key_b64, - site_controller_pid: runtime.site_controller.as_ref().map(Child::id), site_controller_url: plan.site_controller_url.clone(), last_error, } @@ -1471,7 +1445,6 @@ pub(super) fn persist_site_state( .clone(), router_identity_id: launched.receipt.router_identity_id.clone(), router_public_key_b64: launched.receipt.router_public_key_b64.clone(), - site_controller_pid: launched.receipt.site_controller_pid, site_controller_url: launched.receipt.site_controller_url.clone(), last_error, }, @@ -2308,7 +2281,6 @@ mod tests { kubernetes_consumer_router_mesh_addr: Some("192.168.65.254:24000".to_string()), router_identity_id: Some(router_identity.id.clone()), router_public_key_b64: None, - site_controller_pid: None, site_controller_url: None, }, router_control: ControlEndpoint::Tcp("127.0.0.1:24100".to_string()), @@ -2392,7 +2364,6 @@ mod tests { kubernetes_consumer_router_mesh_addr: Some("192.168.65.254:24000".to_string()), router_identity_id: Some(router_identity.id.clone()), router_public_key_b64: None, - site_controller_pid: None, site_controller_url: None, }, router_control: ControlEndpoint::Tcp("127.0.0.1:24100".to_string()), diff --git a/cli/src/mixed_run/tests.rs b/cli/src/mixed_run/tests.rs index d617e91a..dece2640 100644 --- a/cli/src/mixed_run/tests.rs +++ b/cli/src/mixed_run/tests.rs @@ -71,7 +71,6 @@ fn recorded_process_roots_include_site_supervisor() { kubernetes_consumer_router_mesh_addr: None, router_identity_id: None, router_public_key_b64: None, - site_controller_pid: None, site_controller_url: None, }; @@ -239,7 +238,6 @@ fn local_site_controller_addr_requires_loopback_http() { port_forward_mesh_port: None, port_forward_control_port: None, observability_endpoint: None, - site_controller_plan_path: None, site_controller_url: Some("http://127.0.0.1:24200".to_string()), launch_env: BTreeMap::new(), }; @@ -313,7 +311,6 @@ fn local_site_controller_ready_waits_for_http_listener() { port_forward_mesh_port: None, port_forward_control_port: None, observability_endpoint: None, - site_controller_plan_path: None, site_controller_url: Some(format!("http://127.0.0.1:{}", addr.port())), launch_env: BTreeMap::new(), }; @@ -361,7 +358,6 @@ fn compose_site_controller_container_name_uses_compose_project() { port_forward_mesh_port: None, port_forward_control_port: None, observability_endpoint: None, - site_controller_plan_path: None, site_controller_url: Some("http://amber-site-controller:4100".to_string()), launch_env: BTreeMap::new(), }; @@ -372,6 +368,40 @@ fn compose_site_controller_container_name_uses_compose_project() { ); } +#[test] +fn compose_site_without_controller_has_no_controller_container_name() { + let plan = SiteSupervisorPlan { + schema: "amber.run.site_supervisor_plan".to_string(), + version: 2, + run_id: "run".to_string(), + mesh_scope: "scope".to_string(), + run_root: "/tmp/run".to_string(), + coordinator_pid: 1, + site_id: "compose-site".to_string(), + kind: SiteKind::Compose, + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), + storage_root: None, + runtime_root: None, + router_mesh_port: None, + compose_project: Some("amber_run_compose-site".to_string()), + kubernetes_namespace: None, + context: None, + port_forward_mesh_port: None, + port_forward_control_port: None, + observability_endpoint: None, + site_controller_url: None, + launch_env: BTreeMap::new(), + }; + + assert_eq!( + supervisor::compose_site_controller_container_name(&plan), + None, + "compose sites without a lowered controller should not be treated as if they had a \ + special controller container" + ); +} + #[test] fn reserve_loopback_port_keeps_allocations_unique_within_one_process() { let mut ports = BTreeSet::new(); diff --git a/cli/src/run_inputs.rs b/cli/src/run_inputs.rs index 43cbd9c4..04c83792 100644 --- a/cli/src/run_inputs.rs +++ b/cli/src/run_inputs.rs @@ -1725,6 +1725,7 @@ mod tests { site: SiteDefinition { kind, context: None, + controller_site: None, }, router_identity_id: "/site/test/router".to_string(), assigned_components: vec!["/app".to_string()], @@ -1805,6 +1806,7 @@ mod tests { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults::default(), diff --git a/cli/src/tests.rs b/cli/src/tests.rs index 8460cd0c..b02fb860 100644 --- a/cli/src/tests.rs +++ b/cli/src/tests.rs @@ -1200,43 +1200,6 @@ fn rewrite_peer_addr_for_slirp_gateway_rewrites_loopback_only() { ); } -#[cfg(target_os = "linux")] -#[test] -fn rewrite_loopback_url_for_slirp_gateway_rewrites_loopback_only() { - assert_eq!( - rewrite_loopback_url_for_slirp_gateway("http://127.0.0.1:23000/base?x=1"), - "http://10.0.2.2:23000/base?x=1" - ); - assert_eq!( - rewrite_loopback_url_for_slirp_gateway("http://localhost:24000"), - "http://10.0.2.2:24000/" - ); - assert_eq!( - rewrite_loopback_url_for_slirp_gateway("http://192.168.1.10:25000/path"), - "http://192.168.1.10:25000/path" - ); - assert_eq!( - rewrite_loopback_url_for_slirp_gateway("not-a-url"), - "not-a-url" - ); -} - -#[cfg(target_os = "linux")] -#[test] -fn rewrite_sidecar_env_passthrough_for_slirp_rewrites_only_controller_url() { - assert_eq!( - rewrite_sidecar_env_passthrough_for_slirp( - amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, - "http://127.0.0.1:25000/v1/control-state" - ), - "http://10.0.2.2:25000/v1/control-state" - ); - assert_eq!( - rewrite_sidecar_env_passthrough_for_slirp("UNRELATED_ENV", "http://127.0.0.1:25000"), - "http://127.0.0.1:25000" - ); -} - #[cfg(target_os = "linux")] #[test] fn slirp4netns_add_hostfwd_payload_uses_guest_default_address() { diff --git a/cli/src/vm_runtime/preview.rs b/cli/src/vm_runtime/preview.rs index 219080bf..5aa2cd97 100644 --- a/cli/src/vm_runtime/preview.rs +++ b/cli/src/vm_runtime/preview.rs @@ -138,14 +138,40 @@ pub(crate) fn build_vm_site_launch_preview( mesh_plan, } = load_vm_runtime_inputs(&plan_path)?; - let port_assignments = materialize_vm_runtime( + let mut inspectability_warnings = Vec::new(); + let port_assignments = match materialize_vm_runtime( &plan_root, runtime_root, &vm_plan, &mesh_plan, router_mesh_port, true, - )?; + ) { + Ok(port_assignments) => port_assignments, + Err(err) if crate::direct_runtime::missing_existing_peer_identity(&err) => { + // Dry-run preview can still show the local VM launch shape before peer-site routers + // exist; the missing identities only block full mesh config materialization. + inspectability_warnings.push(format!( + "preview is missing one or more peer-site router identities, so mesh configs \ + remain unresolved until those sites are running: {err}" + )); + let (preview_peer_identities, preview_peer_ports) = + crate::direct_runtime::preview_placeholder_peer_mesh_state(&mesh_plan); + materialize_vm_runtime_with_existing( + &plan_root, + runtime_root, + &vm_plan, + &mesh_plan, + router_mesh_port, + VmExistingMeshState { + reuse_existing: false, + peer_ports_by_id: &preview_peer_ports, + peer_identities_by_id: &preview_peer_identities, + }, + )? + } + Err(err) => return Err(err), + }; let arch = host_arch()?; let (qemu_system, qemu_warning) = preview_qemu_system_binary(arch); let (amber_cli, amber_warning) = preview_amber_binary(); @@ -169,7 +195,6 @@ pub(crate) fn build_vm_site_launch_preview( .transpose()?; let mut virtual_machines = Vec::new(); - let mut inspectability_warnings = Vec::new(); inspectability_warnings.extend(qemu_warning); inspectability_warnings.extend(amber_warning); for component_id in &vm_plan.startup_order { diff --git a/cli/src/vm_runtime/tests.rs b/cli/src/vm_runtime/tests.rs index 69726913..93172d47 100644 --- a/cli/src/vm_runtime/tests.rs +++ b/cli/src/vm_runtime/tests.rs @@ -1,5 +1,11 @@ -use amber_compiler::reporter::vm::{MESH_PROVISION_PLAN_FILENAME, VmStorageMount}; -use amber_mesh::{MeshProtocol, OutboundRoute, TransportConfig}; +use amber_compiler::reporter::vm::{ + MESH_PROVISION_PLAN_FILENAME, VM_PLAN_FILENAME, VmStorageMount, +}; +use amber_mesh::{ + MESH_PROVISION_PLAN_VERSION, MeshConfigTemplate, MeshIdentityTemplate, MeshPeerTemplate, + MeshProtocol, MeshProvisionOutput, MeshProvisionPlan, MeshProvisionTarget, + MeshProvisionTargetKind, OutboundRoute, TransportConfig, +}; use super::*; @@ -449,6 +455,104 @@ fn build_vm_launch_preview_keeps_command_when_base_image_is_unresolved() { })); } +#[test] +fn build_vm_site_launch_preview_tolerates_missing_peer_router_identity() { + let temp = tempfile::tempdir().expect("temp dir"); + let plan_root = temp.path().join("artifact"); + let runtime_root = temp.path().join("runtime"); + let storage_root = temp.path().join("storage"); + fs::create_dir_all(&plan_root).expect("plan root"); + fs::create_dir_all(&runtime_root).expect("runtime root"); + fs::create_dir_all(&storage_root).expect("storage root"); + let base_image = temp.path().join("base.img"); + fs::write(&base_image, []).expect("base image"); + + let vm_plan = VmPlan { + version: VM_PLAN_VERSION.to_string(), + mesh_provision_plan: MESH_PROVISION_PLAN_FILENAME.to_string(), + startup_order: vec![7], + runtime_addresses: DirectRuntimeAddressPlan::default(), + components: vec![VmComponentPlan { + id: 7, + moniker: "/app".to_string(), + log_name: "app".to_string(), + depends_on: Vec::new(), + mesh_config_path: "mesh/components/app/mesh-config.json".to_string(), + mesh_identity_path: "mesh/components/app/mesh-identity.json".to_string(), + cpus: VmScalarPlanU32::Literal { value: 1 }, + memory_mib: VmScalarPlanU32::Literal { value: 512 }, + base_image: VmHostPathPlan::Static { + path: base_image.display().to_string(), + }, + cloud_init_user_data: None, + cloud_init_vendor_data: None, + egress: VmEgressPlan::None, + storage_mounts: Vec::new(), + runtime_config: None, + mount_spec_b64: None, + }], + router: None, + }; + let mesh_plan = MeshProvisionPlan { + version: MESH_PROVISION_PLAN_VERSION.to_string(), + identity_seed: None, + existing_peer_identities: Vec::new(), + targets: vec![MeshProvisionTarget { + kind: MeshProvisionTargetKind::Component, + config: MeshConfigTemplate { + identity: MeshIdentityTemplate { + id: "/app".to_string(), + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:19000".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: vec![MeshPeerTemplate { + id: "/site/direct_local/router".to_string(), + }], + inbound: Vec::new(), + outbound: Vec::new(), + transport: TransportConfig::NoiseIk {}, + }, + output: MeshProvisionOutput::Filesystem { + dir: "mesh/components/app".to_string(), + }, + }], + }; + + fs::write( + plan_root.join(VM_PLAN_FILENAME), + serde_json::to_string_pretty(&vm_plan).expect("serialize vm plan"), + ) + .expect("write vm plan"); + fs::write( + plan_root.join(MESH_PROVISION_PLAN_FILENAME), + serde_json::to_string_pretty(&mesh_plan).expect("serialize mesh provision plan"), + ) + .expect("write mesh provision plan"); + + let preview = build_vm_site_launch_preview( + &plan_root.join(VM_PLAN_FILENAME), + &storage_root, + &runtime_root, + None, + ) + .expect("vm preview"); + + assert_eq!(preview.virtual_machines.len(), 1); + assert!( + !preview.virtual_machines[0].command.is_empty(), + "preview should still emit a local QEMU command when peer routers are unresolved" + ); + assert!( + preview + .inspectability_warnings + .iter() + .any(|warning| warning.contains("missing one or more peer-site router identities")) + ); +} + #[test] fn render_cloud_init_multipart_keeps_generated_and_user_parts() { let rendered = render_cloud_init_multipart( diff --git a/cli/tests/mixed_run/smoke.rs b/cli/tests/mixed_run/smoke.rs index e2b1c5b7..e2949737 100644 --- a/cli/tests/mixed_run/smoke.rs +++ b/cli/tests/mixed_run/smoke.rs @@ -817,9 +817,6 @@ fn framework_controller_post(run: &RunHandle, path: &str, payload: &Value) -> (u let authority_url = plan["authority_url"] .as_str() .expect("site controller plan should publish authority_url"); - let auth_token = plan["auth_token"] - .as_str() - .expect("site controller plan should publish auth token"); let body = serde_json::to_string(payload).expect("request body should serialize"); let output = command_output_via_tempfiles( std::process::Command::new("curl") @@ -831,7 +828,22 @@ fn framework_controller_post(run: &RunHandle, path: &str, payload: &Value) -> (u .arg("-H") .arg("content-type: application/json") .arg("-H") - .arg(format!("x-amber-framework-auth: {auth_token}")) + .arg("-H") + .arg("x-amber-site-controller-local-only: 1") + .arg("-H") + .arg(format!( + "x-amber-route-id: site-controller:{}", + plan["site_id"] + .as_str() + .expect("site controller plan should publish site_id") + )) + .arg("-H") + .arg(format!( + "x-amber-peer-id: {}", + plan["router_identity_id"] + .as_str() + .expect("site controller plan should publish router identity") + )) .arg("--data") .arg(body) .arg("-o") diff --git a/cli/tests/mixed_run/support.rs b/cli/tests/mixed_run/support.rs index e26409c0..955109f4 100644 --- a/cli/tests/mixed_run/support.rs +++ b/cli/tests/mixed_run/support.rs @@ -190,8 +190,6 @@ struct StaleSiteReceipt { process_pid: Option, #[serde(default)] port_forward_pid: Option, - #[serde(default)] - site_controller_pid: Option, } impl TestTempDir { @@ -317,7 +315,6 @@ fn stale_run_has_live_processes(receipt: &StaleRunReceipt) -> bool { Some(site.supervisor_pid), site.process_pid, site.port_forward_pid, - site.site_controller_pid, ] }) .flatten(), diff --git a/compiler/src/reporter/mod.rs b/compiler/src/reporter/mod.rs index 605458da..eced8b90 100644 --- a/compiler/src/reporter/mod.rs +++ b/compiler/src/reporter/mod.rs @@ -86,7 +86,32 @@ impl CompiledScenario { pub fn derive_from_ir(&self, scenario_ir: ScenarioIr) -> Result { let mut compiled = Self::from_ir(scenario_ir)?; + let required_slots = compiled + .scenario + .components + .len() + .max(compiled.scenario.root.0 + 1); compiled.resolved_urls = self.resolved_urls.clone(); + if compiled.resolved_urls.len() < required_slots { + compiled.resolved_urls.resize(required_slots, None); + } + let previous_components = self + .scenario_ir + .components + .iter() + .map(|component| (component.id, component)) + .collect::>(); + for component in &mut compiled.scenario_ir.components { + let Some(previous) = previous_components.get(&component.id) else { + continue; + }; + if component.resolved_url.is_none() { + component.resolved_url = previous.resolved_url.clone(); + } + if component.exports.is_empty() { + component.exports = previous.exports.clone(); + } + } compiled.source_context = self.source_context.clone(); Ok(compiled) } diff --git a/compiler/src/run_plan.rs b/compiler/src/run_plan.rs index 1bc8a3df..e3b130c3 100644 --- a/compiler/src/run_plan.rs +++ b/compiler/src/run_plan.rs @@ -164,6 +164,8 @@ pub struct SiteDefinition { pub kind: SiteKind, #[serde(default, skip_serializing_if = "Option::is_none")] pub context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub controller_site: Option, } #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] @@ -175,6 +177,98 @@ pub enum SiteKind { Kubernetes, } +pub const FRAMEWORK_COMPONENT_CONTROLLER_METADATA_KIND: &str = + "amber.framework_component.controller"; +pub const FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_IMAGE: &str = "__amber_internal/site-controller"; +pub const FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PATH: &str = "/__amber_internal/site-controller"; +pub const FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE: &str = + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FrameworkComponentControllerMetadata { + pub kind: String, + pub execution_site: String, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub grants: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FrameworkComponentGrantMetadata { + pub managed_site: String, + pub authority_realm_id: usize, + pub authority_realm_moniker: String, +} + +pub fn framework_component_controller_metadata( + metadata: Option<&Value>, +) -> Option { + metadata + .cloned() + .and_then(|value| { + serde_json::from_value::(value).ok() + }) + .filter(|metadata| metadata.kind == FRAMEWORK_COMPONENT_CONTROLLER_METADATA_KIND) +} + +pub(crate) fn lower_framework_component_bindings_for_single_site( + compiled: &CompiledScenario, + site_id: &str, + site: SiteDefinition, +) -> Result { + let scenario = compiled.scenario(); + let assignments_by_component = scenario + .components_iter() + .map(|(component_id, _)| (component_id, site_id.to_string())) + .collect::>(); + let offered_sites = BTreeMap::from([(site_id.to_string(), site)]); + let controller_sites = if has_public_framework_component_bindings(scenario) { + BTreeSet::from([site_id.to_string()]) + } else { + BTreeSet::new() + }; + let lowered = inject_site_controller_components( + scenario, + &offered_sites, + &assignments_by_component, + &controller_sites, + )?; + compiled + .derive_from_ir(ScenarioIr::from(&lowered.scenario)) + .map_err(|err| { + RunPlanError::Other(format!( + "failed to lower framework.component bindings for site `{site_id}`: {err}" + )) + }) +} + +pub fn lower_framework_component_bindings_for_placement( + compiled: &CompiledScenario, + placement: &PlacementFile, +) -> Result { + let scenario = compiled.scenario(); + let offered_sites = normalize_site_definitions(&placement_site_definitions(Some(placement)))?; + let assignments_by_component = resolve_assignments( + scenario, + &offered_sites, + &placement.defaults, + placement.components.clone(), + )?; + validate_storage_locality(scenario, &assignments_by_component)?; + let lowered = inject_site_controller_components( + scenario, + &offered_sites, + &assignments_by_component, + &BTreeSet::new(), + )?; + compiled + .derive_from_ir(ScenarioIr::from(&lowered.scenario)) + .map_err(|err| { + RunPlanError::Other(format!( + "failed to lower framework.component bindings for placement-aware scenario: {err}" + )) + }) +} + #[derive(Debug, Error)] pub enum PlacementParseError { #[error("invalid placement file: {0}")] @@ -263,30 +357,18 @@ pub fn build_run_plan_with_activation( placement: Option<&PlacementFile>, activation_override: Option<&RunPlanActivationState>, ) -> Result { - let scenario = compiled.scenario(); - let offered_sites = placement_site_definitions(placement); + let offered_sites = normalize_site_definitions(&placement_site_definitions(placement))?; let defaults = placement .map(|placement| placement.defaults.clone()) .unwrap_or_else(default_placement_defaults); - let endpoint_plan = build_endpoint_plan(scenario) - .map_err(|err| RunPlanError::Other(format!("failed to build endpoint plan: {err}")))?; - let mesh_plan = build_mesh_plan( - scenario, - &endpoint_plan, - MeshOptions { - backend_label: "run plan", - }, - ) - .map_err(|err| RunPlanError::Other(format!("failed to build mesh plan: {err}")))?; - let placement_components = placement_component_overrides(placement); - let assignments_by_component = resolve_assignments( - scenario, + let mut assignments_by_component = resolve_assignments( + compiled.scenario(), &offered_sites, &defaults, placement_components.clone(), )?; - validate_storage_locality(scenario, &assignments_by_component)?; + validate_storage_locality(compiled.scenario(), &assignments_by_component)?; let ( standby_sites, @@ -304,13 +386,16 @@ pub fn build_run_plan_with_activation( activation.active_site_capabilities.clone(), ) } else { - let standby_sites = analyze_standby_sites(scenario, &offered_sites, &defaults)? + let standby_sites = analyze_standby_sites(compiled.scenario(), &offered_sites, &defaults)? .into_iter() .collect::>(); - let static_active_sites = assignments_by_component - .values() - .cloned() - .collect::>(); + let static_active_sites = expand_sites_to_execution_sites( + &assignments_by_component + .values() + .cloned() + .collect::>(), + &offered_sites, + )?; let standby_site_set = standby_sites.iter().cloned().collect::>(); let control_only_sites = Vec::new(); let control_only_site_set = control_only_sites.iter().cloned().collect::>(); @@ -348,6 +433,45 @@ pub fn build_run_plan_with_activation( active_site_capabilities, ) }; + + let controller_sites = if has_public_framework_component_bindings(compiled.scenario()) { + active_site_capabilities + .keys() + .map(|site_id| framework_controller_execution_site_id(&offered_sites, site_id)) + .collect::, _>>()? + .into_iter() + .map(str::to_string) + .collect::>() + } else { + BTreeSet::new() + }; + let lowered = inject_site_controller_components( + compiled.scenario(), + &offered_sites, + &assignments_by_component, + &controller_sites, + )?; + assignments_by_component.extend(lowered.synthetic_assignments.clone()); + let lowered_compiled = compiled + .derive_from_ir(ScenarioIr::from(&lowered.scenario)) + .map_err(|err| { + RunPlanError::Other(format!( + "failed to lower framework.component bindings: {err}" + )) + })?; + let compiled = &lowered_compiled; + let scenario = compiled.scenario(); + let endpoint_plan = build_endpoint_plan(scenario) + .map_err(|err| RunPlanError::Other(format!("failed to build endpoint plan: {err}")))?; + let mesh_plan = build_mesh_plan( + scenario, + &endpoint_plan, + MeshOptions { + backend_label: "run plan", + }, + ) + .map_err(|err| RunPlanError::Other(format!("failed to build mesh plan: {err}")))?; + let initial_active_site_set = initial_active_sites .iter() .map(String::as_str) @@ -504,6 +628,26 @@ fn validate_activation_override( }); } } + + let initial_active_sites = activation + .initial_active_sites + .iter() + .cloned() + .collect::>(); + for site_id in activation + .standby_sites + .iter() + .chain(&activation.initial_active_sites) + { + let execution_site = site_execution_site_id(offered_sites, site_id)?; + if initial_active_sites.contains(execution_site) { + continue; + } + return Err(RunPlanError::Other(format!( + "activation state marks site `{site_id}` active, but omits its controller execution \ + site `{execution_site}` from initial_active_sites" + ))); + } Ok(()) } @@ -520,6 +664,7 @@ pub fn build_homogeneous_export_run_plan( SiteDefinition { kind: requested_kind, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -615,11 +760,7 @@ pub fn plan_dynamic_fragment( .flat_map(|link| [link.provider_site.clone(), link.consumer_site.clone()]), ) .collect::>(); - let site_plans = render_site_plan_subset( - scenario, - &planning, - affected_sites.iter().map(String::as_str), - )?; + let site_plans = render_site_plan_subset(&planning, affected_sites.iter().map(String::as_str))?; let assignments = fragment_components .iter() .filter_map(|component_id| { @@ -650,7 +791,7 @@ pub fn build_site_plan_subset( ) -> Result, RunPlanError> { let planning = resolve_dynamic_planning_state(scenario, placement, activation, existing_assignments)?; - render_site_plan_subset(scenario, &planning, site_ids.iter().map(String::as_str)) + render_site_plan_subset(&planning, site_ids.iter().map(String::as_str)) } fn resolve_assignments( @@ -701,6 +842,7 @@ fn resolve_assignments( #[derive(Clone, Debug)] struct DynamicPlanningState { + scenario: Scenario, mesh_scope: String, offered_sites: BTreeMap, active_site_capabilities: BTreeMap, @@ -714,18 +856,38 @@ fn resolve_dynamic_planning_state( activation: &RunPlanActivationState, existing_assignments: &BTreeMap, ) -> Result { - let offered_sites = placement_site_definitions(Some(placement)); + let offered_sites = normalize_site_definitions(&placement_site_definitions(Some(placement)))?; validate_activation_override(&offered_sites, activation)?; let mut explicit_components = placement.components.clone(); explicit_components.extend(existing_assignments.clone()); - let assignments_by_component = resolve_assignments( + let mut assignments_by_component = resolve_assignments( scenario, &offered_sites, &placement.defaults, explicit_components, )?; validate_storage_locality(scenario, &assignments_by_component)?; + let controller_sites = if has_public_framework_component_bindings(scenario) { + activation + .active_site_capabilities + .keys() + .map(|site_id| framework_controller_execution_site_id(&offered_sites, site_id)) + .collect::, _>>()? + .into_iter() + .map(str::to_string) + .collect::>() + } else { + BTreeSet::new() + }; + let lowered = inject_site_controller_components( + scenario, + &offered_sites, + &assignments_by_component, + &controller_sites, + )?; + assignments_by_component.extend(lowered.synthetic_assignments.clone()); + let scenario = lowered.scenario; let active_sites = activation .dynamic_enabled_sites @@ -737,25 +899,26 @@ fn resolve_dynamic_planning_state( .find(|site_id| !active_sites.contains(site_id.as_str())) { return Err(RunPlanError::Other(format!( - "component placement selected site `{site_id}`, but the frozen activation state does \ - not allow dynamic workloads there" - ))); + "component placement selected site `{site_id}`, but the frozen activation state \ + does not allow dynamic workloads there" + ))); } - let endpoint_plan = build_endpoint_plan(scenario) + let endpoint_plan = build_endpoint_plan(&scenario) .map_err(|err| RunPlanError::Other(format!("failed to build endpoint plan: {err}")))?; let mesh_plan = build_mesh_plan( - scenario, + &scenario, &endpoint_plan, MeshOptions { backend_label: "dynamic plan", }, ) .map_err(|err| RunPlanError::Other(format!("failed to build dynamic mesh plan: {err}")))?; - let links = build_cross_site_links(scenario, &mesh_plan, &assignments_by_component); + let links = build_cross_site_links(&scenario, &mesh_plan, &assignments_by_component); Ok(DynamicPlanningState { - mesh_scope: scenario_mesh_scope(&ScenarioIr::from(scenario))?, + scenario: scenario.clone(), + mesh_scope: scenario_mesh_scope(&ScenarioIr::from(&scenario))?, offered_sites, active_site_capabilities: activation.active_site_capabilities.clone(), assignments_by_component, @@ -764,10 +927,10 @@ fn resolve_dynamic_planning_state( } fn render_site_plan_subset<'a>( - scenario: &Scenario, planning: &DynamicPlanningState, site_ids: impl IntoIterator, ) -> Result, RunPlanError> { + let scenario = &planning.scenario; let links_by_provider_site = planning .links @@ -968,6 +1131,7 @@ fn default_site_definitions() -> BTreeMap { SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, ), ( @@ -975,6 +1139,7 @@ fn default_site_definitions() -> BTreeMap { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ( @@ -982,11 +1147,55 @@ fn default_site_definitions() -> BTreeMap { SiteDefinition { kind: SiteKind::Vm, context: None, + controller_site: None, }, ), ]) } +fn normalize_site_definitions( + site_definitions: &BTreeMap, +) -> Result, RunPlanError> { + let direct_sites = site_definitions + .iter() + .filter_map(|(site_id, site)| (site.kind == SiteKind::Direct).then_some(site_id.clone())) + .collect::>(); + let mut normalized = site_definitions.clone(); + + for (site_id, site) in &mut normalized { + if site.kind != SiteKind::Vm { + if site.controller_site.is_some() { + return Err(RunPlanError::Other(format!( + "site `{site_id}` sets `controller_site`, but only vm sites support explicit \ + controller managers" + ))); + } + continue; + } + + if let Some(controller_site) = site.controller_site.as_deref() { + let controller = site_definitions.get(controller_site).ok_or_else(|| { + RunPlanError::Other(format!( + "vm site `{site_id}` references unknown controller_site `{controller_site}`" + )) + })?; + if controller.kind != SiteKind::Direct { + return Err(RunPlanError::Other(format!( + "vm site `{site_id}` must use a direct controller_site, but \ + `{controller_site}` is {:?}", + controller.kind + ))); + } + continue; + } + if direct_sites.len() == 1 { + site.controller_site = Some(direct_sites[0].clone()); + } + } + + Ok(normalized) +} + fn default_placement_defaults() -> PlacementDefaults { PlacementDefaults { path: Some("direct_local".to_string()), @@ -995,6 +1204,53 @@ fn default_placement_defaults() -> PlacementDefaults { } } +fn site_execution_site_id<'a>( + site_definitions: &'a BTreeMap, + site_id: &'a str, +) -> Result<&'a str, RunPlanError> { + let site = site_definitions + .get(site_id) + .ok_or_else(|| RunPlanError::UnknownSite { + site_id: site_id.to_string(), + })?; + match site.kind { + SiteKind::Vm => Ok(site.controller_site.as_deref().unwrap_or(site_id)), + SiteKind::Direct | SiteKind::Compose | SiteKind::Kubernetes => Ok(site_id), + } +} + +fn framework_controller_execution_site_id<'a>( + site_definitions: &'a BTreeMap, + site_id: &'a str, +) -> Result<&'a str, RunPlanError> { + let site = site_definitions + .get(site_id) + .ok_or_else(|| RunPlanError::UnknownSite { + site_id: site_id.to_string(), + })?; + match site.kind { + SiteKind::Vm => site.controller_site.as_deref().ok_or_else(|| { + RunPlanError::Other(format!( + "vm site `{site_id}` requires an explicit controller_site because no controlling \ + direct site is available for framework.component" + )) + }), + SiteKind::Direct | SiteKind::Compose | SiteKind::Kubernetes => Ok(site_id), + } +} + +fn expand_sites_to_execution_sites( + site_ids: &BTreeSet, + site_definitions: &BTreeMap, +) -> Result, RunPlanError> { + let mut expanded = BTreeSet::new(); + for site_id in site_ids { + expanded.insert(site_id.clone()); + expanded.insert(site_execution_site_id(site_definitions, site_id)?.to_string()); + } + Ok(expanded) +} + #[derive(Clone, Debug)] struct FrozenChildTemplateSpec { manifests: Option>, @@ -1040,7 +1296,7 @@ fn analyze_standby_sites( } } - Ok(requested_sites) + expand_sites_to_execution_sites(&requested_sites, site_definitions) } fn collect_frozen_child_templates( @@ -1894,63 +2150,469 @@ fn render_site_artifact_files( .collect()) } -#[cfg(test)] -mod tests { - use std::{ - collections::{BTreeMap, BTreeSet}, - fs, - path::Path, - }; +#[derive(Clone, Debug)] +struct LoweredFrameworkComponentScenario { + scenario: Scenario, + synthetic_assignments: BTreeMap, +} - use amber_manifest::ManifestRef; - use amber_resolver::Resolver; - use tempfile::TempDir; - use url::Url; +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +struct FrameworkComponentGrantKey { + managed_site: String, + authority_realm: ComponentId, + authority_realm_moniker: String, +} - use super::*; - use crate::{ - CompileOptions, Compiler, DigestStore, ResolverRegistry, reporter::CompiledScenario, - }; +#[derive(Clone, Debug)] +struct ControllerComponentRewrite { + component_id: ComponentId, + provide_by_grant: BTreeMap, +} - fn tmp_dir(prefix: &str) -> TempDir { - tempfile::Builder::new().prefix(prefix).tempdir().unwrap() - } +const FRAMEWORK_COMPONENT_CONTROLLER_ENDPOINT_NAME: &str = "framework_component"; +const FRAMEWORK_COMPONENT_CONTROLLER_MONIKER_PREFIX: &str = + "/__amber_internal_framework_component_controller"; +const FRAMEWORK_COMPONENT_CONTROLLER_PROVIDE_PREFIX: &str = "__amber_internal_framework_component"; +const FRAMEWORK_COMPONENT_CONTROLLER_PORT_BASE: u16 = 32000; +const FRAMEWORK_COMPONENT_CONTROLLER_CONTAINER_PATH: &str = "/usr/local/bin/amber-site-controller"; +const FRAMEWORK_COMPONENT_CONTROLLER_PLAN_PATH: &str = + "/amber/site/state/site-controller-plan.json"; + +fn has_public_framework_component_bindings(scenario: &Scenario) -> bool { + scenario.bindings.iter().any(|binding| { + matches!( + &binding.from, + BindingFrom::Framework(framework) if framework.capability.as_str() == "component" + ) + }) +} - fn write(path: &Path, contents: &str) { - fs::write(path, contents).unwrap(); +fn inject_site_controller_components( + scenario: &Scenario, + offered_sites: &BTreeMap, + assignments_by_component: &BTreeMap, + controller_sites: &BTreeSet, +) -> Result { + let framework_bindings = scenario + .bindings + .iter() + .filter_map(|binding| match &binding.from { + BindingFrom::Framework(framework) if framework.capability.as_str() == "component" => { + Some((binding, framework)) + } + _ => None, + }) + .collect::>(); + if framework_bindings.is_empty() { + return Ok(LoweredFrameworkComponentScenario { + scenario: scenario.clone(), + synthetic_assignments: BTreeMap::new(), + }); } - fn manifest_ref(path: &Path) -> ManifestRef { - ManifestRef::from_url(Url::from_file_path(path).unwrap()) + let moniker_to_site = assignments_by_component + .iter() + .map(|(component_id, site_id)| { + ( + graph::component_path(scenario, *component_id), + site_id.clone(), + ) + }) + .collect::>(); + let mut grants_by_execution_site = + BTreeMap::>::new(); + for (binding, framework) in &framework_bindings { + let managed_site = framework_component_managed_site_id( + scenario, + &moniker_to_site, + framework.authority, + binding.to.component, + )?; + let execution_site = + framework_controller_execution_site_id(offered_sites, &managed_site)?.to_string(); + grants_by_execution_site + .entry(execution_site) + .or_default() + .insert(FrameworkComponentGrantKey { + managed_site, + authority_realm: framework.authority, + authority_realm_moniker: graph::component_path(scenario, framework.authority), + }); } - fn image_server_manifest() -> &'static str { - r#"{ - manifest_version: "0.3.0", - program: { - image: "busybox:1.36.1", - entrypoint: ["sh", "-c", "sleep 30"], - network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] } - }, - provides: { api: { kind: "http", endpoint: "http" } }, - exports: { api: "api" } -}"# + let mut used_ports = scenario + .components_iter() + .flat_map(|(_, component)| { + component + .program + .as_ref() + .and_then(|program| program.network()) + .into_iter() + .flat_map(|network| network.endpoints.iter().map(|endpoint| endpoint.port)) + }) + .collect::>(); + let mut rewritten = scenario.clone(); + let mut synthetic_assignments = BTreeMap::new(); + let mut controller_rewrites = BTreeMap::::new(); + + let execution_sites = controller_sites + .iter() + .cloned() + .chain(grants_by_execution_site.keys().cloned()) + .collect::>(); + for execution_site in execution_sites { + let grants = grants_by_execution_site + .remove(&execution_site) + .unwrap_or_default(); + let controller_id = ComponentId(rewritten.components.len()); + let port = next_framework_component_controller_port(&mut used_ports)?; + let site = offered_sites + .get(&execution_site) + .ok_or_else(|| RunPlanError::UnknownSite { + site_id: execution_site.clone(), + })?; + let mut grants_metadata = BTreeMap::new(); + let mut provides = BTreeMap::from([( + FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE.to_string(), + framework_component_controller_internal_provide_decl()?, + )]); + let mut provide_by_grant = BTreeMap::new(); + for grant in grants { + let provide_name = framework_component_controller_provide_name(&grant); + grants_metadata.insert( + provide_name.clone(), + FrameworkComponentGrantMetadata { + managed_site: grant.managed_site.clone(), + authority_realm_id: grant.authority_realm.0, + authority_realm_moniker: grant.authority_realm_moniker.clone(), + }, + ); + provides.insert( + provide_name.clone(), + framework_component_controller_provide_decl()?, + ); + provide_by_grant.insert(grant, provide_name); + } + let metadata = FrameworkComponentControllerMetadata { + kind: FRAMEWORK_COMPONENT_CONTROLLER_METADATA_KIND.to_string(), + execution_site: execution_site.clone(), + grants: grants_metadata, + }; + rewritten.components.push(Some(Component { + id: controller_id, + parent: Some(rewritten.root), + moniker: unique_framework_component_controller_moniker(&rewritten, &execution_site) + .into(), + digest: rewritten.component(rewritten.root).digest, + config: None, + config_schema: None, + program: Some(framework_component_controller_program(site.kind, port)?), + slots: BTreeMap::new(), + provides, + resources: BTreeMap::new(), + metadata: Some( + serde_json::to_value(metadata) + .expect("framework controller metadata should serialize"), + ), + child_templates: BTreeMap::new(), + children: Vec::new(), + })); + rewritten + .component_mut(rewritten.root) + .children + .push(controller_id); + synthetic_assignments.insert(controller_id, execution_site.clone()); + controller_rewrites.insert( + execution_site, + ControllerComponentRewrite { + component_id: controller_id, + provide_by_grant, + }, + ); } - fn path_server_manifest() -> &'static str { - r#"{ - manifest_version: "0.3.0", - program: { - path: "/usr/bin/env", - args: ["sh", "-c", "sleep 30"], - network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] } - }, - provides: { api: { kind: "http", endpoint: "http" } }, - exports: { api: "api" } -}"# + for binding in &mut rewritten.bindings { + let BindingFrom::Framework(framework) = &binding.from else { + continue; + }; + if framework.capability.as_str() != "component" { + continue; + } + let grant = FrameworkComponentGrantKey { + managed_site: framework_component_managed_site_id( + scenario, + &moniker_to_site, + framework.authority, + binding.to.component, + )?, + authority_realm: framework.authority, + authority_realm_moniker: graph::component_path(scenario, framework.authority), + }; + let execution_site = + framework_controller_execution_site_id(offered_sites, &grant.managed_site)?; + let rewrite = controller_rewrites.get(execution_site).expect( + "synthetic controller rewrite should exist for each lowered framework.component grant", + ); + let provide = rewrite + .provide_by_grant + .get(&grant) + .expect("lowered framework.component grant should have a synthetic provide"); + binding.from = BindingFrom::Component(ProvideRef { + component: rewrite.component_id, + name: provide.clone(), + }); } - fn vm_server_manifest() -> &'static str { + rewritten.normalize_order(); + rewritten.assert_invariants(); + Ok(LoweredFrameworkComponentScenario { + scenario: rewritten, + synthetic_assignments, + }) +} + +fn framework_component_managed_site_id( + scenario: &Scenario, + moniker_to_site: &BTreeMap, + authority_realm: ComponentId, + recipient_component: ComponentId, +) -> Result { + let authority_moniker = graph::component_path(scenario, authority_realm); + if let Some(site_id) = moniker_to_site.get(&authority_moniker) { + return Ok(site_id.clone()); + } + + let recipient_moniker = graph::component_path(scenario, recipient_component); + if authority_moniker != "/" + && recipient_moniker != authority_moniker + && !recipient_moniker.starts_with(&format!("{authority_moniker}/")) + { + return Err(RunPlanError::Other(format!( + "framework.component recipient `{recipient_moniker}` is not inside authority \ + realm `{authority_moniker}`" + ))); + } + + let recipient_segments = recipient_moniker + .trim_start_matches('/') + .split('/') + .filter(|segment| !segment.is_empty()) + .collect::>(); + let authority_depth = authority_moniker + .trim_matches('/') + .split('/') + .filter(|segment| !segment.is_empty()) + .count(); + for depth in authority_depth..=recipient_segments.len() { + let moniker = if depth == 0 { + "/".to_string() + } else { + format!("/{}", recipient_segments[..depth].join("/")) + }; + if let Some(site_id) = moniker_to_site.get(&moniker) { + return Ok(site_id.clone()); + } + } + + Err(RunPlanError::Other(format!( + "framework.component authority realm `{authority_moniker}` has no assigned component \ + on the path to recipient `{recipient_moniker}`" + ))) +} + +fn framework_component_controller_provide_name(grant: &FrameworkComponentGrantKey) -> String { + format!( + "{FRAMEWORK_COMPONENT_CONTROLLER_PROVIDE_PREFIX}__site_{}__authority_{}", + sanitize_framework_component_identifier(grant.managed_site.as_str()), + grant.authority_realm.0, + ) +} + +fn unique_framework_component_controller_moniker( + scenario: &Scenario, + execution_site: &str, +) -> String { + let base = format!( + "{FRAMEWORK_COMPONENT_CONTROLLER_MONIKER_PREFIX}/{}", + sanitize_framework_component_identifier(execution_site), + ); + let used = scenario + .components_iter() + .map(|(_, component)| component.moniker.as_str()) + .collect::>(); + if !used.contains(base.as_str()) { + return base; + } + let mut suffix = 1usize; + loop { + let candidate = format!("{base}-{suffix}"); + if !used.contains(candidate.as_str()) { + return candidate; + } + suffix += 1; + } +} + +fn sanitize_framework_component_identifier(raw: &str) -> String { + let mut out = String::with_capacity(raw.len()); + for ch in raw.chars() { + if ch.is_ascii_alphanumeric() { + out.push(ch.to_ascii_lowercase()); + } else { + out.push('_'); + } + } + if out.is_empty() { + "internal".to_string() + } else { + out + } +} + +fn next_framework_component_controller_port( + used_ports: &mut BTreeSet, +) -> Result { + for port in FRAMEWORK_COMPONENT_CONTROLLER_PORT_BASE..=u16::MAX { + if used_ports.insert(port) { + return Ok(port); + } + } + Err(RunPlanError::Other( + "unable to allocate a synthetic framework.component controller port".to_string(), + )) +} + +fn framework_component_controller_program( + kind: SiteKind, + port: u16, +) -> Result { + let value = match kind { + SiteKind::Direct => serde_json::json!({ + "path": FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PATH, + "network": { + "endpoints": [ + { + "name": FRAMEWORK_COMPONENT_CONTROLLER_ENDPOINT_NAME, + "port": port, + "protocol": "http" + } + ] + } + }), + SiteKind::Compose | SiteKind::Kubernetes => serde_json::json!({ + "image": FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_IMAGE, + "entrypoint": [ + FRAMEWORK_COMPONENT_CONTROLLER_CONTAINER_PATH, + "--plan", + FRAMEWORK_COMPONENT_CONTROLLER_PLAN_PATH + ], + "network": { + "endpoints": [ + { + "name": FRAMEWORK_COMPONENT_CONTROLLER_ENDPOINT_NAME, + "port": port, + "protocol": "http" + } + ] + } + }), + SiteKind::Vm => { + return Err(RunPlanError::Other( + "synthetic framework.component controllers must execute on non-vm sites" + .to_string(), + )); + } + }; + serde_json::from_value(value).map_err(|err| { + RunPlanError::Other(format!( + "failed to build synthetic framework.component controller program: {err}" + )) + }) +} + +fn framework_component_controller_provide_decl() -> Result +{ + serde_json::from_value(serde_json::json!({ + "kind": "component", + "endpoint": FRAMEWORK_COMPONENT_CONTROLLER_ENDPOINT_NAME, + })) + .map_err(|err| { + RunPlanError::Other(format!( + "failed to build synthetic framework.component controller provide: {err}" + )) + }) +} + +fn framework_component_controller_internal_provide_decl() +-> Result { + serde_json::from_value(serde_json::json!({ + "kind": "http", + "endpoint": FRAMEWORK_COMPONENT_CONTROLLER_ENDPOINT_NAME, + })) + .map_err(|err| { + RunPlanError::Other(format!( + "failed to build synthetic site controller internal provide: {err}" + )) + }) +} + +#[cfg(test)] +mod tests { + use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + path::Path, + }; + + use amber_manifest::ManifestRef; + use amber_resolver::Resolver; + use tempfile::TempDir; + use url::Url; + + use super::*; + use crate::{ + CompileOptions, Compiler, DigestStore, ResolverRegistry, reporter::CompiledScenario, + }; + + fn tmp_dir(prefix: &str) -> TempDir { + tempfile::Builder::new().prefix(prefix).tempdir().unwrap() + } + + fn write(path: &Path, contents: &str) { + fs::write(path, contents).unwrap(); + } + + fn manifest_ref(path: &Path) -> ManifestRef { + ManifestRef::from_url(Url::from_file_path(path).unwrap()) + } + + fn image_server_manifest() -> &'static str { + r#"{ + manifest_version: "0.3.0", + program: { + image: "busybox:1.36.1", + entrypoint: ["sh", "-c", "sleep 30"], + network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] } + }, + provides: { api: { kind: "http", endpoint: "http" } }, + exports: { api: "api" } +}"# + } + + fn path_server_manifest() -> &'static str { + r#"{ + manifest_version: "0.3.0", + program: { + path: "/usr/bin/env", + args: ["sh", "-c", "sleep 30"], + network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] } + }, + provides: { api: { kind: "http", endpoint: "http" } }, + exports: { api: "api" } +}"# + } + + fn vm_server_manifest() -> &'static str { r##"{ manifest_version: "0.3.0", program: { @@ -2111,6 +2773,114 @@ mod tests { }"# } + fn framework_admin_path_manifest() -> &'static str { + r#"{ + manifest_version: "0.3.0", + slots: { ctl: { kind: "component" } }, + program: { + path: "/usr/bin/env", + args: ["sh", "-c", "sleep 30"], + env: { CTL_URL: "${slots.ctl.url}" }, + network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] } + }, + provides: { api: { kind: "http", endpoint: "http" } }, + exports: { api: "api" } +}"# + } + + fn framework_admin_vm_manifest() -> &'static str { + r##"{ + manifest_version: "0.3.0", + slots: { ctl: { kind: "component" } }, + program: { + vm: { + image: "ubuntu.img", + cpus: 1, + memory_mib: 512, + network: { + endpoints: [{ name: "http", port: 8080, protocol: "http" }], + egress: "none" + }, + cloud_init: { + user_data: "#cloud-config\nwrite_files:\n - path: /etc/ctl-url\n content: '${slots.ctl.url}'\nruncmd:\n - [sh, -lc, 'sleep infinity']\n" + } + } + }, + provides: { api: { kind: "http", endpoint: "http" } }, + exports: { api: "api" } +}"## + } + + fn framework_authority_manifest() -> &'static str { + r##"{ + manifest_version: "0.3.0", + components: { admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + program: { + image: "busybox:1.36.1", + entrypoint: ["sh", "-c", "sleep 30"], + network: { endpoints: [{ name: "http", port: 8081, protocol: "http" }] } + }, + provides: { api: { kind: "http", endpoint: "http" } }, + exports: { + api: "api", + admin_api: "#admin.api" + } +}"## + } + + fn framework_controller_component_for_site<'a>( + scenario: &'a Scenario, + execution_site: &str, + ) -> ( + ComponentId, + &'a Component, + FrameworkComponentControllerMetadata, + ) { + if let Some(controller) = + scenario + .components_iter() + .find_map(|(component_id, component)| { + framework_component_controller_metadata(component.metadata.as_ref()) + .filter(|metadata| metadata.execution_site == execution_site) + .map(|metadata| (component_id, component, metadata)) + }) + { + return controller; + } + + let seen = scenario + .components_iter() + .map(|(_, component)| { + format!( + "{} metadata={:?}", + component.moniker.as_str(), + component.metadata.as_ref() + ) + }) + .collect::>() + .join("\n"); + let bindings = scenario + .bindings + .iter() + .map(|binding| { + format!( + "to={}.{} from={:?}", + graph::component_path(scenario, binding.to.component), + binding.to.name, + binding.from + ) + }) + .collect::>() + .join("\n"); + panic!( + "synthetic framework.component controller for site `{execution_site}` should be \ + present;\ncomponents:\n{seen}\nbindings:\n{bindings}" + ) + } + async fn compile(root: &Path) -> CompiledScenario { let compiler = Compiler::new(Resolver::new(), DigestStore::default()) .with_registry(ResolverRegistry::default()); @@ -2127,6 +2897,487 @@ mod tests { CompiledScenario::from_compile_output(&output).expect("compiled scenario") } + #[tokio::test] + async fn vm_site_defaults_controller_site_to_sole_direct_site() { + let dir = tmp_dir("run-plan-vm-controller-default-"); + let vm_child = dir.path().join("vm.json5"); + let root = dir.path().join("root.json5"); + + write(&vm_child, vm_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { vm: "./vm.json5" }, + exports: { vm_api: "#vm.api" } +}"##, + ); + + let compiled = compile(&root).await; + let placement = parse_placement_file( + r#"{ + schema: "amber.run.placement", + version: 1, + sites: { + host_direct: { kind: "direct" }, + guest_vm: { kind: "vm" } + }, + defaults: { + vm: "guest_vm" + } +}"#, + ) + .expect("placement should parse"); + let plan = build_run_plan(&compiled, Some(&placement)).expect("run plan should build"); + assert_eq!( + plan.offered_sites["guest_vm"].controller_site.as_deref(), + Some("host_direct") + ); + assert_eq!( + plan.sites["guest_vm"].site.controller_site.as_deref(), + Some("host_direct") + ); + } + + #[tokio::test] + async fn framework_component_vm_sites_require_explicit_controller_site_when_multiple_direct_sites_exist() + { + let dir = tmp_dir("run-plan-vm-controller-ambiguous-"); + let admin = dir.path().join("admin.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_vm_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + exports: { admin_api: "#admin.api" } +}"##, + ); + + let compiled = compile(&root).await; + let placement = parse_placement_file( + r#"{ + schema: "amber.run.placement", + version: 1, + sites: { + host_a: { kind: "direct" }, + host_b: { kind: "direct" }, + guest_vm: { kind: "vm" } + }, + defaults: { + path: "host_a", + image: "host_a", + vm: "guest_vm" + } +}"#, + ) + .expect("placement should parse"); + let err = build_run_plan(&compiled, Some(&placement)).expect_err("run plan should fail"); + let rendered = err.to_string(); + assert!(rendered.contains("guest_vm"), "{rendered}"); + assert!(rendered.contains("controller_site"), "{rendered}"); + assert!(rendered.contains("framework.component"), "{rendered}"); + } + + #[tokio::test] + async fn vm_site_rejects_non_direct_controller_site() { + let dir = tmp_dir("run-plan-vm-controller-kind-"); + let vm_child = dir.path().join("vm.json5"); + let root = dir.path().join("root.json5"); + + write(&vm_child, vm_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { vm: "./vm.json5" }, + exports: { vm_api: "#vm.api" } +}"##, + ); + + let placement = parse_placement_file( + r#"{ + schema: "amber.run.placement", + version: 1, + sites: { + compose_mgr: { kind: "compose" }, + guest_vm: { kind: "vm", controller_site: "compose_mgr" } + }, + defaults: { + vm: "guest_vm" + } +}"#, + ) + .expect("placement should parse"); + let compiled = compile(&root).await; + let err = build_run_plan(&compiled, Some(&placement)).expect_err("run plan should fail"); + let rendered = err.to_string(); + assert!(rendered.contains("guest_vm"), "{rendered}"); + assert!(rendered.contains("compose_mgr"), "{rendered}"); + assert!(rendered.contains("direct controller_site"), "{rendered}"); + } + + #[tokio::test] + async fn vm_site_preserves_explicit_controller_site() { + let dir = tmp_dir("run-plan-vm-controller-explicit-"); + let vm_child = dir.path().join("vm.json5"); + let root = dir.path().join("root.json5"); + + write(&vm_child, vm_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { vm: "./vm.json5" }, + exports: { vm_api: "#vm.api" } +}"##, + ); + + let placement = parse_placement_file( + r#"{ + schema: "amber.run.placement", + version: 1, + sites: { + host_a: { kind: "direct" }, + host_b: { kind: "direct" }, + guest_vm: { kind: "vm", controller_site: "host_b" } + }, + defaults: { + vm: "guest_vm" + } +}"#, + ) + .expect("placement should parse"); + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, Some(&placement)).expect("run plan should build"); + assert_eq!( + plan.offered_sites["guest_vm"].controller_site.as_deref(), + Some("host_b") + ); + } + + #[tokio::test] + async fn vm_static_workloads_activate_their_controller_site() { + let dir = tmp_dir("run-plan-vm-controller-active-"); + let vm_child = dir.path().join("vm.json5"); + let root = dir.path().join("root.json5"); + + write(&vm_child, vm_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { vm: "./vm.json5" }, + exports: { vm_api: "#vm.api" } +}"##, + ); + + let placement = parse_placement_file( + r#"{ + schema: "amber.run.placement", + version: 1, + sites: { + host_direct: { kind: "direct" }, + guest_vm: { kind: "vm" } + }, + defaults: { + vm: "guest_vm" + } +}"#, + ) + .expect("placement should parse"); + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, Some(&placement)).expect("run plan should build"); + assert_eq!( + plan.initial_active_sites, + vec!["guest_vm".to_string(), "host_direct".to_string()] + ); + assert_eq!(plan.dynamic_enabled_sites, plan.initial_active_sites); + assert!(plan.sites.contains_key("host_direct")); + assert!(plan.sites.contains_key("guest_vm")); + assert_eq!( + plan.sites["host_direct"].assigned_components, + Vec::::new() + ); + assert_eq!( + plan.sites["guest_vm"].assigned_components, + vec!["/vm".to_string()] + ); + } + + #[tokio::test] + async fn vm_standby_sites_activate_their_controller_site() { + let dir = tmp_dir("run-plan-vm-controller-standby-"); + let worker = dir.path().join("worker.json5"); + let root = dir.path().join("root.json5"); + + write(&worker, vm_server_manifest()); + write(&root, dynamic_parent_manifest()); + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, None).expect("run plan should build"); + assert_eq!( + plan.standby_sites, + vec!["direct_local".to_string(), "vm_local".to_string()] + ); + assert_eq!( + plan.initial_active_sites, + vec![ + "compose_local".to_string(), + "direct_local".to_string(), + "vm_local".to_string(), + ] + ); + assert_eq!(plan.dynamic_enabled_sites, plan.initial_active_sites); + } + + #[tokio::test] + async fn sites_without_public_framework_bindings_do_not_inject_site_controller_components() { + let dir = tmp_dir("run-plan-controller-injection-"); + let worker = dir.path().join("worker.json5"); + let root = dir.path().join("root.json5"); + + write(&worker, path_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { worker: "./worker.json5" }, + exports: { worker_api: "#worker.api" } +}"##, + ); + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, None).expect("run plan should build"); + let site_plan = plan + .sites + .get("direct_local") + .expect("direct site should be present"); + let scenario = Scenario::try_from(site_plan.scenario_ir.clone()) + .expect("site scenario should round-trip from IR"); + + assert!( + scenario.components_iter().all(|(_, component)| { + framework_component_controller_metadata(component.metadata.as_ref()).is_none() + }), + "sites without public framework.component bindings should not inject a controller" + ); + } + + #[tokio::test] + async fn framework_component_lowering_rewrites_same_site_bindings_to_synthetic_controller_components() + { + let dir = tmp_dir("run-plan-framework-component-direct-"); + let admin = dir.path().join("admin.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_path_manifest()); + write( + &root, + r###"{ + manifest_version: "0.3.0", + components: { admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + exports: { admin_api: "#admin.api" } +}"###, + ); + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, None).expect("run plan should build"); + let lowered = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered base scenario should deserialize"); + let scenario = lowered.scenario(); + let (controller_id, _, metadata) = + framework_controller_component_for_site(scenario, "direct_local"); + assert_eq!(metadata.grants.len(), 1); + let binding = scenario + .bindings + .iter() + .find(|binding| { + graph::component_path(scenario, binding.to.component) == "/admin" + && binding.to.name == "ctl" + }) + .expect("framework binding should remain present on /admin.ctl"); + let BindingFrom::Component(provide) = &binding.from else { + panic!("framework binding should lower to a synthetic controller provide"); + }; + assert_eq!(provide.component, controller_id); + let grant = metadata + .grants + .get(&provide.name) + .expect("lowered controller provide should carry grant metadata"); + assert_eq!(grant.managed_site, "direct_local"); + assert_eq!(grant.authority_realm_moniker, "/"); + assert!(scenario.bindings.iter().all( + |binding| !matches!(&binding.from, BindingFrom::Framework(framework) if framework.capability.as_str() == "component") + )); + assert!(plan.links.is_empty()); + } + + #[tokio::test] + async fn framework_component_lowering_routes_cross_site_authorities_to_their_managed_site_controller() + { + let dir = tmp_dir("run-plan-framework-component-cross-site-"); + let admin = dir.path().join("admin.json5"); + let authority = dir.path().join("authority.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_path_manifest()); + write(&authority, framework_authority_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { authority: "./authority.json5" }, + exports: { + authority_api: "#authority.api", + authority_admin_api: "#authority.admin_api" + } +}"##, + ); + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, None).expect("run plan should build"); + let lowered = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered base scenario should deserialize"); + let scenario = lowered.scenario(); + let (controller_id, controller, metadata) = + framework_controller_component_for_site(scenario, "compose_local"); + let binding = scenario + .bindings + .iter() + .find(|binding| { + graph::component_path(scenario, binding.to.component) == "/authority/admin" + && binding.to.name == "ctl" + }) + .expect("nested framework binding should remain present"); + let BindingFrom::Component(provide) = &binding.from else { + panic!("nested framework binding should lower to a synthetic controller provide"); + }; + assert_eq!(provide.component, controller_id); + let grant = metadata + .grants + .get(&provide.name) + .expect("lowered controller provide should carry grant metadata"); + assert_eq!(grant.managed_site, "compose_local"); + assert_eq!(grant.authority_realm_moniker, "/authority"); + let ctl_link = plan + .links + .iter() + .find(|link| link.consumer_component == "/authority/admin" && link.slot == "ctl") + .expect("cross-site framework binding should become an ordinary cross-site link"); + assert_eq!(ctl_link.provider_site, "compose_local"); + assert_eq!(ctl_link.consumer_site, "direct_local"); + assert_eq!(ctl_link.provider_component, controller.moniker.as_str()); + } + + #[tokio::test] + async fn framework_component_lowering_executes_vm_managers_on_their_direct_controller_site() { + let dir = tmp_dir("run-plan-framework-component-vm-"); + let admin = dir.path().join("admin.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_vm_manifest()); + write( + &root, + r###"{ + manifest_version: "0.3.0", + components: { admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + exports: { admin_api: "#admin.api" } +}"###, + ); + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, None).expect("run plan should build"); + let lowered = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered base scenario should deserialize"); + let scenario = lowered.scenario(); + let (controller_id, controller, metadata) = + framework_controller_component_for_site(scenario, "direct_local"); + let binding = scenario + .bindings + .iter() + .find(|binding| { + graph::component_path(scenario, binding.to.component) == "/admin" + && binding.to.name == "ctl" + }) + .expect("vm framework binding should remain present"); + let BindingFrom::Component(provide) = &binding.from else { + panic!("vm framework binding should lower to a synthetic controller provide"); + }; + assert_eq!(provide.component, controller_id); + let grant = metadata + .grants + .get(&provide.name) + .expect("vm controller provide should carry grant metadata"); + assert_eq!(grant.managed_site, "vm_local"); + let ctl_link = plan + .links + .iter() + .find(|link| link.consumer_component == "/admin" && link.slot == "ctl") + .expect("vm framework binding should become a cross-site link to the direct manager"); + assert_eq!(ctl_link.provider_site, "direct_local"); + assert_eq!(ctl_link.consumer_site, "vm_local"); + assert_eq!(ctl_link.provider_component, controller.moniker.as_str()); + } + + #[tokio::test] + async fn compiled_scenario_derive_from_ir_extends_resolved_url_slots_for_synthetic_components() + { + let dir = tmp_dir("run-plan-derived-ir-slots-"); + let admin = dir.path().join("admin.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, path_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { admin: "./admin.json5" }, + exports: { admin_api: "#admin.api" } +}"##, + ); + + let compiled = compile(&root).await; + let mut scenario = compiled.scenario().clone(); + let synthetic_id = ComponentId(scenario.components.len()); + scenario.components.push(Some(Component { + id: synthetic_id, + parent: Some(scenario.root), + moniker: "/__amber_internal_test".to_string().into(), + digest: scenario.component(scenario.root).digest, + config: None, + config_schema: None, + program: None, + slots: BTreeMap::new(), + provides: BTreeMap::new(), + resources: BTreeMap::new(), + metadata: None, + child_templates: BTreeMap::new(), + children: Vec::new(), + })); + scenario + .component_mut(scenario.root) + .children + .push(synthetic_id); + scenario.normalize_order(); + + let derived = compiled + .derive_from_ir(ScenarioIr::from(&scenario)) + .expect("derived compiled scenario should allow injected components"); + assert!(derived.resolved_url_for_component(synthetic_id).is_none()); + } + #[tokio::test] async fn default_placement_prefers_compose_for_images() { let dir = tmp_dir("run-plan-default-placement-"); diff --git a/compiler/src/targets/direct/mod.rs b/compiler/src/targets/direct/mod.rs index 7023f71f..19511233 100644 --- a/compiler/src/targets/direct/mod.rs +++ b/compiler/src/targets/direct/mod.rs @@ -19,6 +19,10 @@ use crate::{ GENERATED_ENV_SAMPLE_FILENAME, GENERATED_README_FILENAME, build_execution_guide, }, }, + run_plan::{ + SiteDefinition, SiteKind, framework_component_controller_metadata, + lower_framework_component_bindings_for_single_site, + }, targets::{ common::{TargetError as MeshError, component_label}, mesh::{ @@ -170,6 +174,7 @@ pub enum DirectProgramExecutionPlan { #[serde(default, skip_serializing_if = "Option::is_none")] mount_spec_b64: Option, }, + InternalSiteController, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -230,6 +235,17 @@ pub(crate) fn emit_direct_artifact_with_options( compiled: &CompiledScenario, options: DirectArtifactBuildOptions<'_>, ) -> Result { + let lowered = lower_framework_component_bindings_for_single_site( + compiled, + "direct_local", + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ) + .map_err(|err| MeshError::new(err.to_string()))?; + let compiled = &lowered; let scenario = compiled.scenario(); let endpoint_plan = crate::targets::program_config::build_endpoint_plan(scenario)?; let mesh_plan = build_mesh_plan( @@ -490,11 +506,15 @@ fn build_component_plans( .map(|deps| deps.iter().map(|dep| dep.0).collect::>()) .unwrap_or_default(); let source_dir = component_source_dir(compiled, *id, component.moniker.as_str())?; - let execution = resolve_direct_execution_plan( - direct_execution_plan(runtime_plan.execution), - source_dir.as_deref(), - component.moniker.as_str(), - )?; + let execution = if let Some(execution) = direct_internal_execution_plan(component) { + execution + } else { + resolve_direct_execution_plan( + direct_execution_plan(runtime_plan.execution), + source_dir.as_deref(), + component.moniker.as_str(), + )? + }; out.push(DirectComponentPlan { id: id.0, @@ -524,6 +544,13 @@ fn build_component_plans( Ok(out) } +fn direct_internal_execution_plan( + component: &amber_scenario::Component, +) -> Option { + framework_component_controller_metadata(component.metadata.as_ref()) + .map(|_| DirectProgramExecutionPlan::InternalSiteController) +} + fn inject_direct_dynamic_caps_env( execution: DirectProgramExecutionPlan, dynamic_caps_port: u16, @@ -552,6 +579,9 @@ fn inject_direct_dynamic_caps_env( runtime_config, mount_spec_b64, }, + DirectProgramExecutionPlan::InternalSiteController => { + DirectProgramExecutionPlan::InternalSiteController + } } } @@ -705,6 +735,9 @@ fn resolve_direct_execution_plan( runtime_config, mount_spec_b64, }), + DirectProgramExecutionPlan::InternalSiteController => { + Ok(DirectProgramExecutionPlan::InternalSiteController) + } } } @@ -1109,9 +1142,6 @@ mod tests { }; use amber_manifest::{Manifest, ManifestRef}; - use amber_mesh::{ - FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV, FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, - }; use amber_resolver::Resolver; use amber_scenario::{BindingEdge, Component, Moniker, Scenario}; use tempfile::TempDir; @@ -1195,7 +1225,7 @@ mod tests { } #[test] - fn direct_router_passthrough_includes_framework_controller_auth() { + fn direct_framework_component_is_lowered_to_an_internal_site_controller_component() { let dir = TempDir::new().expect("temp dir"); let root_path = dir.path().join("root.json5"); let admin_path = dir.path().join("admin.json5"); @@ -1224,9 +1254,6 @@ mod tests { r##" {{ manifest_version: "0.3.0", - slots: {{ - realm: {{ kind: "component", optional: true }} - }}, components: {{ admin: "{admin}" }}, @@ -1260,23 +1287,13 @@ mod tests { .expect("direct plan should be emitted"), ) .expect("direct plan should deserialize"); - let router = direct_plan - .router - .expect("framework.component binding should force a router"); assert!( - router - .env_passthrough - .iter() - .any(|env_var| env_var == FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV), - "router must receive the framework controller URL env passthrough", - ); - assert!( - router - .env_passthrough - .iter() - .any(|env_var| env_var == FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV), - "router must receive the framework controller auth env passthrough", + direct_plan.components.iter().any(|component| matches!( + component.program.execution, + DirectProgramExecutionPlan::InternalSiteController + )), + "framework.component should lower to an injected internal site controller component" ); } diff --git a/compiler/src/targets/mesh/addressing.rs b/compiler/src/targets/mesh/addressing.rs index 205fd5d6..c0f27810 100644 --- a/compiler/src/targets/mesh/addressing.rs +++ b/compiler/src/targets/mesh/addressing.rs @@ -184,10 +184,11 @@ impl Addressing for LocalAddressing<'_> { binding: &ResolvedFrameworkBinding, ) -> Result { match binding.capability.as_str() { - "component" => { - let local_port = self.local_framework_binding_port(binding)?; - Ok(format!("http://127.0.0.1:{local_port}")) - } + "component" => Err(MeshError::new(format!( + "framework.component binding {}.{} must be lowered before mesh address planning", + component_label(self.scenario, binding.consumer), + binding.slot, + ))), "docker" => match self.options.docker_binding { DockerFrameworkBindingPolicy::LoopbackTcp => { let local_port = self.local_framework_binding_port(binding)?; @@ -592,7 +593,7 @@ mod tests { } #[test] - fn build_address_plan_maps_framework_component_to_loopback_http() { + fn build_address_plan_rejects_unlowered_framework_component() { let mut consumer = component(0, "/consumer", "consumer"); consumer.slots.insert( "realm".to_string(), @@ -634,16 +635,12 @@ mod tests { docker_binding: DockerFrameworkBindingPolicy::LoopbackTcp, }, ); - let plan = build_address_plan(&mesh_plan, addressing).expect("address plan"); - - let slot_value = plan - .slot_values_by_component - .get(&ComponentId(0)) - .and_then(|slots| slots.get("realm")) - .expect("slot value"); - let SlotValue::One(value) = slot_value else { - panic!("expected singular slot value, got {slot_value:?}"); - }; - assert_eq!(value.url, "http://127.0.0.1:20000"); + let err = build_address_plan(&mesh_plan, addressing) + .expect_err("framework.component should be lowered before address planning"); + assert!( + err.to_string() + .contains("must be lowered before mesh address planning"), + "unexpected address-plan error: {err}", + ); } } diff --git a/compiler/src/targets/mesh/docker_compose/mod.rs b/compiler/src/targets/mesh/docker_compose/mod.rs index 8e2c398b..b9558156 100644 --- a/compiler/src/targets/mesh/docker_compose/mod.rs +++ b/compiler/src/targets/mesh/docker_compose/mod.rs @@ -29,6 +29,10 @@ use crate::{ build_execution_guide, }, }, + run_plan::{ + SiteDefinition, SiteKind, framework_component_controller_metadata, + lower_framework_component_bindings_for_single_site, + }, runtime_interface::{RootInputDescriptor, collect_root_inputs}, targets::{ mesh::{ @@ -59,6 +63,7 @@ const BOUNDARY_NETWORK_NAME: &str = "amber_boundary"; const ROUTER_SERVICE_NAME: &str = "amber-router"; const ROUTER_CONTROL_INIT_SERVICE_NAME: &str = "amber-router-control-init"; +const SITE_CONTROLLER_SERVICE_NAME: &str = "amber-site-controller"; const PROVISIONER_SERVICE_NAME: &str = "amber-provisioner"; const HELPER_VOLUME_NAME: &str = "amber-helper-bin"; const HELPER_INIT_SERVICE: &str = "amber-init"; @@ -393,8 +398,18 @@ pub(crate) fn emit_docker_compose_artifact_with_options( compiled: &CompiledScenario, options: DockerComposeArtifactBuildOptions<'_>, ) -> Result { + let lowered = lower_framework_component_bindings_for_single_site( + compiled, + "compose_local", + SiteDefinition { + kind: SiteKind::Compose, + context: None, + controller_site: None, + }, + ) + .map_err(|err| ReporterError::new(err.to_string()))?; render_docker_compose_inner( - compiled.scenario(), + lowered.scenario(), options.force_router, options.router_identity_id, options.mesh_scope, @@ -428,7 +443,13 @@ fn render_docker_compose_inner( // Precompute service names (injective & stable). let names: HashMap = map_program_components(s, program_components, |id, local_name| { - let base = service_base_name(id, local_name); + let base = if framework_component_controller_metadata(s.component(id).metadata.as_ref()) + .is_some() + { + SITE_CONTROLLER_SERVICE_NAME.to_string() + } else { + service_base_name(id, local_name) + }; let sidecar = format!("{base}-net"); ServiceNames { program: base.clone(), @@ -857,6 +878,10 @@ fn render_docker_compose_inner( let program_plan = program_plans.get(id).expect("program plan computed"); let image = if Some(*id) == docker_gateway_component { images.docker_gateway.clone() + } else if framework_component_controller_metadata(s.component(*id).metadata.as_ref()) + .is_some() + { + images.site_controller.clone() } else { let image_plan = program_plan.image().ok_or_else(|| { DockerComposeError::Other(format!( diff --git a/compiler/src/targets/mesh/docker_compose/tests/mod.rs b/compiler/src/targets/mesh/docker_compose/tests/mod.rs index 0b9727ee..1c068ce6 100644 --- a/compiler/src/targets/mesh/docker_compose/tests/mod.rs +++ b/compiler/src/targets/mesh/docker_compose/tests/mod.rs @@ -24,6 +24,7 @@ use url::Url; use super::{DockerComposeReporter, *}; use crate::{ + CompileOptions, Compiler, DigestStore, OptimizeOptions, linker::program_lowering::lower_program, targets::{mesh::internal_images::resolve_internal_images, storage::StorageIdentity}, }; @@ -51,6 +52,73 @@ fn render_compose( DockerComposeReporter.emit(&compiled_scenario(output)) } +fn compile_framework_component_output(root_path: &Path) -> crate::CompileOutput { + let compiler = Compiler::new(amber_resolver::Resolver::new(), DigestStore::default()); + let opts = CompileOptions { + optimize: OptimizeOptions { dce: false }, + ..Default::default() + }; + let rt = tokio::runtime::Runtime::new().expect("tokio runtime"); + let url = Url::from_file_path(root_path).expect("file url"); + rt.block_on(compiler.compile(ManifestRef::from_url(url), opts)) + .expect("compile scenario") +} + +#[test] +fn lowered_framework_component_controller_uses_real_internal_image_in_compose_artifacts() { + let dir = tempfile::tempdir().expect("temp dir"); + let root_path = dir.path().join("root.json5"); + let admin_path = dir.path().join("admin.json5"); + fs::write( + &root_path, + r##" + { + manifest_version: "0.1.0", + components: { admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ] + } + "##, + ) + .expect("write root manifest"); + fs::write( + &admin_path, + r#" + { + manifest_version: "0.1.0", + program: { + image: "busybox:1.36.1", + entrypoint: ["sh", "-lc", "sleep 3600"] + }, + slots: { ctl: { kind: "component" } } + } + "#, + ) + .expect("write admin manifest"); + + let output = compile_framework_component_output(&root_path); + let artifact = render_compose(&output).expect("render compose artifact"); + let compose_yaml = artifact.compose_yaml(); + let site_controller_image = internal_images().site_controller; + + assert!( + compose_yaml.contains(&format!("image: {site_controller_image}")), + "synthetic controller components should use the real site-controller image: {compose_yaml}" + ); + assert!( + compose_yaml.contains("amber-site-controller:") + && compose_yaml.contains("amber-site-controller-net:"), + "the lowered controller should render as an ordinary program+sidecar workload: \ + {compose_yaml}" + ); + assert!( + !compose_yaml.contains("__amber_internal/site-controller"), + "compose artifacts must not leak the internal site-controller placeholder image: \ + {compose_yaml}" + ); +} + #[test] fn render_compose_image_uses_root_default_fallback() { let leaves = [rc::SchemaLeaf { diff --git a/compiler/src/targets/mesh/kubernetes/mod.rs b/compiler/src/targets/mesh/kubernetes/mod.rs index e607e7b9..55d5b36a 100644 --- a/compiler/src/targets/mesh/kubernetes/mod.rs +++ b/compiler/src/targets/mesh/kubernetes/mod.rs @@ -26,6 +26,10 @@ use crate::{ CompiledScenario, Reporter, ReporterError, execution_guide::{GENERATED_README_FILENAME, build_execution_guide}, }, + run_plan::{ + SiteDefinition, SiteKind, framework_component_controller_metadata, + lower_framework_component_bindings_for_single_site, + }, runtime_interface::{RootInputDescriptor, build_runtime_interface}, targets::{ mesh::{ @@ -66,6 +70,7 @@ const MESH_CONFIG_DIR: &str = "/amber/mesh"; const MESH_SECRET_VOLUME_NAME: &str = "amber-mesh"; const ROUTER_NAME: &str = "amber-router"; const PROVISIONER_NAME: &str = "amber-provisioner"; +const SITE_CONTROLLER_SERVICE_NAME: &str = "amber-site-controller"; const PROVISIONER_CONFIGMAP_NAME: &str = "amber-mesh-provision"; const PROVISIONER_SERVICE_ACCOUNT: &str = "amber-provisioner"; const PROVISIONER_ROLE_NAME: &str = "amber-provisioner"; @@ -193,6 +198,17 @@ pub(crate) fn emit_kubernetes_artifact_with_options( compiled: &CompiledScenario, options: KubernetesArtifactBuildOptions<'_>, ) -> KubernetesResult { + let lowered = lower_framework_component_bindings_for_single_site( + compiled, + "kubernetes_local", + SiteDefinition { + kind: SiteKind::Kubernetes, + context: None, + controller_site: None, + }, + ) + .map_err(|err| ReporterError::new(err.to_string()))?; + let compiled = &lowered; let s = compiled.scenario(); let scenario_digest = scenario_ir_digest(s).map_err(|err| ReporterError::new(err.to_string()))?; @@ -237,7 +253,13 @@ pub(crate) fn emit_kubernetes_artifact_with_options( let names: HashMap = map_program_components(s, program_components, |id, local_name| { - let base = service_name(id, local_name); + let base = if framework_component_controller_metadata(s.component(id).metadata.as_ref()) + .is_some() + { + SITE_CONTROLLER_SERVICE_NAME.to_string() + } else { + service_name(id, local_name) + }; ComponentNames { service: base.clone(), netpol: format!("{base}-netpol"), @@ -731,6 +753,8 @@ pub(crate) fn emit_kubernetes_artifact_with_options( let mesh_port = *mesh_ports_by_component.get(id).expect("mesh port missing"); let label = component_label(s, *id); let pod_annotations = component_pod_annotations(&label); + let is_site_controller = + framework_component_controller_metadata(s.component(*id).metadata.as_ref()).is_some(); let image_origin = program_plan.image_origin().ok_or_else(|| { ReporterError::new(format!( "internal error: {} is missing a container image origin", @@ -743,14 +767,18 @@ pub(crate) fn emit_kubernetes_artifact_with_options( component_label(s, *id) )) })?; - let image_source = program_image_source(compiled, s, *id, image_origin); - let (program_image, image_source_env_var) = render_kubernetes_image( - image_plan, - &root_leaf_by_path, - &cnames.service, - &label, - image_source.as_ref(), - )?; + let (program_image, image_source_env_var) = if is_site_controller { + (images.site_controller.clone(), None) + } else { + let image_source = program_image_source(compiled, s, *id, image_origin); + render_kubernetes_image( + image_plan, + &root_leaf_by_path, + &cnames.service, + &label, + image_source.as_ref(), + )? + }; let mount_specs = config_plan.mount_specs.get(id).map(Vec::as_slice); let runtime_plan = build_component_runtime_plan( &label, diff --git a/compiler/src/targets/mesh/kubernetes/tests/mod.rs b/compiler/src/targets/mesh/kubernetes/tests/mod.rs index bad07973..99f57d16 100644 --- a/compiler/src/targets/mesh/kubernetes/tests/mod.rs +++ b/compiler/src/targets/mesh/kubernetes/tests/mod.rs @@ -60,6 +60,79 @@ fn render_artifact(output: &crate::CompileOutput) -> super::KubernetesArtifact { .expect("render kubernetes output") } +fn compile_framework_component_output(root_path: &Path) -> crate::CompileOutput { + let compiler = Compiler::new(Resolver::new(), DigestStore::default()); + let opts = CompileOptions { + optimize: OptimizeOptions { dce: false }, + ..Default::default() + }; + let rt = tokio::runtime::Runtime::new().expect("tokio runtime"); + let url = Url::from_file_path(root_path).expect("file url"); + rt.block_on(compiler.compile(ManifestRef::from_url(url), opts)) + .expect("compile scenario") +} + +#[test] +fn lowered_framework_component_controller_uses_real_internal_image_in_kubernetes_artifacts() { + let dir = tempdir().expect("temp dir"); + let root_path = dir.path().join("root.json5"); + let admin_path = dir.path().join("admin.json5"); + fs::write( + &root_path, + r##" + { + manifest_version: "0.1.0", + components: { admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ] + } + "##, + ) + .expect("write root manifest"); + fs::write( + &admin_path, + r#" + { + manifest_version: "0.1.0", + program: { + image: "busybox:1.36.1", + entrypoint: ["sh", "-lc", "sleep 3600"] + }, + slots: { ctl: { kind: "component" } } + } + "#, + ) + .expect("write admin manifest"); + + let output = compile_framework_component_output(&root_path); + let artifact = render_artifact(&output); + let deployment = artifact + .files + .get(&PathBuf::from("03-deployments/amber-site-controller.yaml")) + .expect("site controller deployment"); + let service = artifact + .files + .get(&PathBuf::from("04-services/amber-site-controller.yaml")) + .expect("site controller service"); + let site_controller_image = internal_images().site_controller; + + assert!( + deployment.contains(&site_controller_image), + "synthetic controller deployment should use the real site-controller image: {deployment}" + ); + assert!( + !deployment.contains("__amber_internal/site-controller"), + "kubernetes artifacts must not leak the internal site-controller placeholder image: \ + {deployment}" + ); + assert!( + service.contains("name: amber-site-controller"), + "the lowered controller should render an ordinary service for the controller workload: \ + {service}" + ); +} + fn parse_rendered_env(content: &str) -> std::collections::BTreeMap { content .lines() diff --git a/compiler/src/targets/mesh/kubernetes/tests/rendering.rs b/compiler/src/targets/mesh/kubernetes/tests/rendering.rs index d6bd9310..29f40b1b 100644 --- a/compiler/src/targets/mesh/kubernetes/tests/rendering.rs +++ b/compiler/src/targets/mesh/kubernetes/tests/rendering.rs @@ -780,12 +780,12 @@ fn kubernetes_templates_dynamic_caps_sidecar_control_env() { .get(&PathBuf::from(super::COMPONENT_SIDECAR_ENV_FILE)) .expect("component sidecar env template"); assert!( - sidecar_env.contains("AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL="), - "{sidecar_env}" + !sidecar_env.contains("AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL="), + "controller URLs must not be injected as ambient sidecar env: {sidecar_env}" ); assert!( - sidecar_env.contains("AMBER_FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN="), - "{sidecar_env}" + !sidecar_env.contains("AMBER_FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN="), + "controller auth must not be injected as ambient sidecar env: {sidecar_env}" ); assert!( sidecar_env.contains("AMBER_DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64="), diff --git a/compiler/src/targets/mesh/mesh_config.rs b/compiler/src/targets/mesh/mesh_config.rs index 37e49fb7..49206ed9 100644 --- a/compiler/src/targets/mesh/mesh_config.rs +++ b/compiler/src/targets/mesh/mesh_config.rs @@ -2,21 +2,25 @@ use std::collections::{BTreeSet, HashMap}; use amber_manifest::NetworkProtocol; use amber_mesh::{ - DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV, - FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, InboundRoute, InboundTarget, MeshConfigTemplate, + DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, InboundRoute, InboundTarget, MeshConfigTemplate, MeshIdentityTemplate, MeshPeerTemplate, MeshProtocol, OutboundRoute, component_route_id, - framework_cap_instance_id, http_route_plugins_for_capability_kind, router_export_route_id, - router_external_route_id, telemetry::SCENARIO_RUN_ID_ENV, + http_route_plugins_for_capability_kind, router_export_route_id, router_external_route_id, + telemetry::SCENARIO_RUN_ID_ENV, }; use amber_scenario::{ComponentId, Scenario}; use base64::Engine as _; use sha2::Digest as _; use super::{ - plan::{MeshError, MeshPlan, component_label}, + plan::{EndpointInfo, MeshError, MeshPlan, component_label}, ports::LocalRoutePorts, }; -use crate::runtime_interface::collect_external_slots; +use crate::{ + run_plan::{ + FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE, framework_component_controller_metadata, + }, + runtime_interface::collect_external_slots, +}; pub(crate) const DEFAULT_ROUTER_ID: &str = "/router"; @@ -167,6 +171,10 @@ pub(crate) fn build_mesh_config_plan( None }; + let local_controller_component = mesh_plan.program_components().iter().copied().find(|id| { + framework_component_controller_metadata(scenario.component(*id).metadata.as_ref()).is_some() + }); + let mut consumers_by_provider: HashMap<(ComponentId, String), BTreeSet> = HashMap::new(); for binding in mesh_plan.component_bindings() { @@ -192,14 +200,7 @@ pub(crate) fn build_mesh_config_plan( let mut component_configs: HashMap = HashMap::new(); let mut component_sidecar_env_passthrough = Vec::new(); push_env_passthrough_once(&mut component_sidecar_env_passthrough, SCENARIO_RUN_ID_ENV); - push_env_passthrough_once( - &mut component_sidecar_env_passthrough, - FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, - ); - push_env_passthrough_once( - &mut component_sidecar_env_passthrough, - FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV, - ); + push_env_passthrough_once( &mut component_sidecar_env_passthrough, DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, @@ -218,10 +219,34 @@ pub(crate) fn build_mesh_config_plan( let mut inbound = Vec::new(); for (provide_name, provide_decl) in &scenario.component(id).provides { - let endpoint = mesh_plan - .component_bindings() - .find(|binding| binding.provider == id && binding.provide == *provide_name) - .map(|b| b.endpoint.clone()) + let endpoint = (Some(id) == local_controller_component + && provide_name == FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE) + .then(|| { + provide_decl.endpoint.as_deref().and_then(|endpoint_name| { + scenario + .component(id) + .program + .as_ref() + .and_then(|program| program.network()) + .and_then(|network| { + network + .endpoints + .iter() + .find(|endpoint| endpoint.name == endpoint_name) + }) + .map(|endpoint| EndpointInfo { + port: endpoint.port, + protocol: endpoint.protocol, + }) + }) + }) + .flatten() + .or_else(|| { + mesh_plan + .component_bindings() + .find(|binding| binding.provider == id && binding.provide == *provide_name) + .map(|b| b.endpoint.clone()) + }) .or_else(|| { mesh_plan .exports() @@ -245,6 +270,24 @@ pub(crate) fn build_mesh_config_plan( issuers.insert(consumer_id); } } + if Some(id) == local_controller_component + && provide_name == FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE + { + for consumer in mesh_plan.program_components().iter().copied() { + if consumer == id { + continue; + } + let consumer_id = identities_by_component + .get(&consumer) + .expect("consumer identity missing") + .id + .clone(); + issuers.insert(consumer_id); + } + if let Some(router_identity) = router_identity.as_ref() { + issuers.insert(router_identity.id.clone()); + } + } let route_backs_live_http_authority = !issuers.is_empty() && matches!(protocol, MeshProtocol::Http); if (route_backs_live_http_authority @@ -354,48 +397,49 @@ pub(crate) fn build_mesh_config_plan( }); } - for binding in mesh_plan.framework_bindings() { - if binding.consumer != id || binding.capability.as_str() != "component" { - continue; - } - let listen_port = route_ports.framework_binding_port(binding).ok_or_else(|| { + if let Some(binding) = mesh_plan + .framework_bindings() + .find(|binding| binding.consumer == id && binding.capability.as_str() == "component") + { + return Err(MeshError::new(format!( + "framework.component binding {}.{} must be lowered before mesh config generation", + component_label(scenario, id), + binding.slot, + ))); + } + + if let Some(controller_id) = local_controller_component + && controller_id != id + { + let listen_port = route_ports.controller_internal_port(id).ok_or_else(|| { MeshError::new(format!( - "route port missing for {}.{}", - component_label(scenario, id), - binding.slot + "site controller control route port missing for {}", + component_label(scenario, id) )) })?; - let router_identity = router_identity.as_ref().ok_or_else(|| { - MeshError::new("framework.component bindings require router identity") - })?; - let router_addr = addressing.mesh_addr_for_router()?; - let slot_decl = scenario - .component(binding.consumer) - .slots - .get(binding.slot.as_str()) - .expect("framework binding target slot should exist"); - let authority_moniker = scenario.component(binding.authority_realm).moniker.as_str(); - let consumer_moniker = scenario.component(binding.consumer).moniker.as_str(); - let route_id = framework_cap_instance_id( - authority_moniker, - consumer_moniker, - &binding.consumer.0.to_string(), - &binding.slot, - binding.capability.as_str(), - ); + let peer_addr = addressing.mesh_addr_for_component(controller_id)?; + let peer_id = identities_by_component + .get(&controller_id) + .expect("controller identity missing") + .id + .clone(); outbound.push(OutboundRoute { - route_id, + route_id: component_route_id( + &peer_id, + FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE, + MeshProtocol::Http, + ), rewrite_route_id: None, - slot: binding.slot.clone(), - capability_kind: Some(slot_decl.decl.kind.to_string()), - capability_profile: slot_decl.decl.profile.clone(), + slot: FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE.to_string(), + capability_kind: None, + capability_profile: None, listen_port, listen_addr: None, protocol: MeshProtocol::Http, http_plugins: Vec::new(), - peer_addr: router_addr, - peer_id: router_identity.id.clone(), - capability: binding.capability.to_string(), + peer_addr, + peer_id, + capability: FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE.to_string(), }); } @@ -425,14 +469,6 @@ pub(crate) fn build_mesh_config_plan( let mut router_env_passthrough = Vec::new(); let router_config = if needs_router { - push_env_passthrough_once( - &mut router_env_passthrough, - FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, - ); - push_env_passthrough_once( - &mut router_env_passthrough, - FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV, - ); let router_identity = router_identity.expect("router identity should exist"); let router_ports = router_ports.expect("router ports missing"); let router_mesh_port = router_ports.mesh; @@ -812,7 +848,100 @@ mod tests { } #[test] - fn build_mesh_config_plan_leaves_framework_component_routes_to_site_controller_overlays() { + fn build_mesh_config_plan_opens_controller_internal_route_to_local_sidecars_and_router() { + let consumer = component( + 0, + "/consumer", + json!({ + "image": "consumer", + "entrypoint": ["consumer"], + }), + ); + let mut controller = component( + 1, + "/__amber_internal_framework_component_controller/site-a", + json!({ + "image": "controller", + "entrypoint": ["controller"], + "network": { + "endpoints": [ + { "name": "framework_component", "port": 8080, "protocol": "http" } + ] + } + }), + ); + controller.metadata = Some( + serde_json::to_value(crate::run_plan::FrameworkComponentControllerMetadata { + kind: "amber.framework_component.controller".to_string(), + execution_site: "site-a".to_string(), + grants: BTreeMap::new(), + }) + .expect("controller metadata"), + ); + controller.provides.insert( + FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE.to_string(), + serde_json::from_value::(json!({ + "kind": "component", + "endpoint": "framework_component", + })) + .expect("internal provide decl"), + ); + + let scenario = Scenario { + root: ComponentId(0), + components: vec![Some(consumer), Some(controller)], + bindings: Vec::new(), + exports: Vec::new(), + manifest_catalog: BTreeMap::new(), + }; + let mesh_plan = MeshPlan::new( + vec![ComponentId(0), ComponentId(1)], + Vec::new(), + Vec::new(), + HashMap::new(), + ); + let endpoint_plan = build_endpoint_plan(&scenario).expect("endpoint plan"); + let route_ports = allocate_local_route_ports(&scenario, &endpoint_plan, &mesh_plan) + .expect("local route ports"); + let mesh_ports_by_component = + HashMap::from([(ComponentId(0), 23000), (ComponentId(1), 23001)]); + + let plan = build_mesh_config_plan(MeshConfigBuildInput { + scenario: &scenario, + mesh_plan: &mesh_plan, + route_ports: &route_ports, + mesh_ports_by_component: &mesh_ports_by_component, + router_ports: Some(RouterPorts { + mesh: 24000, + control: 24100, + }), + addressing: &StaticAddressing, + options: MeshConfigBuildOptions { + force_router: true, + router_identity_id: "/site/test/router", + ..default_mesh_config_build_options() + }, + }) + .expect("mesh config plan"); + + let controller_config = plan + .component_configs + .get(&ComponentId(1)) + .expect("controller config"); + let internal_route = controller_config + .inbound + .iter() + .find(|route| route.capability == FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE) + .expect("controller internal inbound route"); + assert_eq!( + internal_route.allowed_issuers, + vec!["/consumer".to_string(), "/site/test/router".to_string()], + "controller internal routing must admit local component sidecars and the site router", + ); + } + + #[test] + fn build_mesh_config_plan_rejects_unlowered_framework_component_bindings() { let mut consumer = component( 0, "/consumer", @@ -855,7 +984,7 @@ mod tests { .expect("local route ports"); let mesh_ports_by_component = HashMap::from([(ComponentId(0), 23000)]); - let plan = build_mesh_config_plan(MeshConfigBuildInput { + let err = build_mesh_config_plan(MeshConfigBuildInput { scenario: &scenario, mesh_plan: &mesh_plan, route_ports: &route_ports, @@ -870,34 +999,12 @@ mod tests { ..default_mesh_config_build_options() }, }) - .expect("mesh config plan"); + .expect_err("framework.component routes must be lowered before mesh config generation"); - let component_config = plan - .component_configs - .get(&ComponentId(0)) - .expect("component config"); assert!( - component_config.outbound.iter().any(|route| { - route.slot == "realm" - && route.capability == "component" - && route.peer_id == "/site/test/router" - && route.peer_addr == "router:24000" - }), - "framework.component consumers should still send requests to the local router", - ); - - let router_config = plan.router_config.expect("router config"); - assert!( - router_config.inbound.iter().all(|route| { - route.capability != "component" - || !matches!( - route.target, - InboundTarget::External { ref url_env, optional } - if url_env == FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV && !optional - ) - }), - "static router mesh config must not bake in framework.component delivery; the site \ - controller overlay owns those routes", + err.to_string() + .contains("must be lowered before mesh config generation"), + "unexpected mesh config error: {err}" ); } } diff --git a/compiler/src/targets/mesh/plan.rs b/compiler/src/targets/mesh/plan.rs index 8df17ee8..40df25ba 100644 --- a/compiler/src/targets/mesh/plan.rs +++ b/compiler/src/targets/mesh/plan.rs @@ -158,12 +158,10 @@ impl MeshPlan { } pub(crate) fn needs_router(&self) -> bool { - self.bindings.iter().any(|binding| { - binding.as_external().is_some() - || binding - .as_framework() - .is_some_and(|binding| binding.capability.as_str() == "component") - }) || !self.exports.is_empty() + self.bindings + .iter() + .any(|binding| binding.as_external().is_some()) + || !self.exports.is_empty() } pub(crate) fn exports(&self) -> &[ResolvedExport] { diff --git a/compiler/src/targets/mesh/ports.rs b/compiler/src/targets/mesh/ports.rs index b6c619be..4bfb6934 100644 --- a/compiler/src/targets/mesh/ports.rs +++ b/compiler/src/targets/mesh/ports.rs @@ -19,6 +19,7 @@ pub(crate) struct LocalRoutePorts { slot_ports_by_component: HashMap>, reserved_ports_by_component: HashMap>, dynamic_caps_ports_by_component: HashMap, + controller_internal_ports_by_component: HashMap, component_binding_ports: HashMap, u16>, external_binding_ports: HashMap, u16>, framework_binding_ports: HashMap, u16>, @@ -45,6 +46,12 @@ impl LocalRoutePorts { .copied() } + pub(crate) fn controller_internal_port(&self, component: ComponentId) -> Option { + self.controller_internal_ports_by_component + .get(&component) + .copied() + } + pub(crate) fn component_binding_port(&self, binding: &ResolvedComponentBinding) -> Option { self.component_binding_ports .get(&BindingIdentity::from(binding)) @@ -198,6 +205,8 @@ pub(crate) fn allocate_local_route_ports( for id in mesh_plan.program_components() { let mut allocator = ComponentRoutePortAllocator::new(endpoint_plan, *id); let dynamic_caps_port = allocator.allocate_reserved(scenario, *id, "dynamic caps api")?; + let controller_internal_port = + allocator.allocate_reserved(scenario, *id, "site controller control route")?; for binding in mesh_plan.bindings_for_consumer(*id) { let port = allocator.allocate(scenario, *id, binding.slot())?; @@ -220,6 +229,8 @@ pub(crate) fn allocate_local_route_ports( let (reserved_ports, slot_ports) = allocator.finish(); out.dynamic_caps_ports_by_component .insert(*id, dynamic_caps_port); + out.controller_internal_ports_by_component + .insert(*id, controller_internal_port); out.reserved_ports_by_component.insert(*id, reserved_ports); out.slot_ports_by_component.insert(*id, slot_ports); } @@ -239,6 +250,9 @@ pub(crate) fn placeholder_local_route_ports( let dynamic_caps_port = allocator .allocate_reserved(scenario, *id, "dynamic caps api") .expect("dynamic caps api port allocation should not overflow"); + let controller_internal_port = allocator + .allocate_reserved(scenario, *id, "site controller control route") + .expect("site controller control route port allocation should not overflow"); for binding in mesh_plan.bindings_for_consumer(*id) { let port = allocator @@ -263,6 +277,8 @@ pub(crate) fn placeholder_local_route_ports( let (reserved, slot_ports) = allocator.finish(); out.dynamic_caps_ports_by_component .insert(*id, dynamic_caps_port); + out.controller_internal_ports_by_component + .insert(*id, controller_internal_port); out.reserved_ports_by_component.insert(*id, reserved); out.slot_ports_by_component.insert(*id, slot_ports); } diff --git a/compiler/src/targets/vm/mod.rs b/compiler/src/targets/vm/mod.rs index e0c69e86..516ea676 100644 --- a/compiler/src/targets/vm/mod.rs +++ b/compiler/src/targets/vm/mod.rs @@ -224,6 +224,18 @@ pub(crate) fn emit_vm_artifact_with_options( options: VmArtifactBuildOptions<'_>, ) -> Result { let scenario = compiled.scenario(); + if scenario.bindings.iter().any(|binding| { + matches!( + &binding.from, + amber_scenario::BindingFrom::Framework(framework) + if framework.capability.as_str() == "component" + ) + }) { + return Err(MeshError::new( + "vm reporter does not support framework.component without an explicit controlling \ + direct site; use mixed-site placement instead", + )); + } let endpoint_plan = crate::targets::program_config::build_endpoint_plan(scenario)?; let mesh_plan = build_mesh_plan( scenario, @@ -1106,3 +1118,88 @@ fn component_source_dir( fn vm_runtime_addresses_is_empty(plan: &DirectRuntimeAddressPlan) -> bool { plan.slots_by_scope.is_empty() && plan.slot_items_by_scope.is_empty() } + +#[cfg(test)] +mod tests { + use std::path::Path; + + use amber_manifest::ManifestRef; + use amber_resolver::Resolver; + use tempfile::TempDir; + use url::Url; + + use super::*; + use crate::{CompileOptions, Compiler, DigestStore}; + + fn write_file(path: &Path, contents: &str) { + std::fs::write(path, contents).expect("fixture file should write"); + } + + fn file_url(path: &Path) -> Url { + Url::from_file_path(path).expect("fixture path should convert to file url") + } + + #[test] + fn vm_reporter_rejects_framework_component_without_a_controller_site() { + let dir = TempDir::new().expect("temp dir"); + let root_path = dir.path().join("root.json5"); + let admin_path = dir.path().join("admin.json5"); + + write_file( + &admin_path, + r#" + { + manifest_version: "0.3.0", + slots: { + ctl: { kind: "component" } + }, + program: { + path: "/bin/echo", + args: ["admin", "${slots.ctl.url}"], + network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] } + }, + provides: { http: { kind: "http", endpoint: "http" } }, + exports: { http: "provides.http" } + } + "#, + ); + write_file( + &root_path, + &format!( + r##" + {{ + manifest_version: "0.3.0", + components: {{ + admin: "{admin}" + }}, + bindings: [ + {{ to: "#admin.ctl", from: "framework.component" }} + ], + exports: {{ + admin_http: "#admin.http" + }} + }} + "##, + admin = file_url(&admin_path), + ), + ); + + let compiler = Compiler::new(Resolver::new(), DigestStore::default()); + let runtime = tokio::runtime::Runtime::new().expect("tokio runtime"); + let output = runtime + .block_on(compiler.compile( + ManifestRef::from_url(file_url(&root_path)), + CompileOptions::default(), + )) + .expect("fixture should compile"); + let compiled = CompiledScenario::from_compile_output(&output) + .expect("fixture should materialize compiled scenario"); + let err = emit_vm_artifact(&compiled, false) + .expect_err("vm reporter should reject framework.component"); + assert!( + err.to_string().contains("explicit controlling direct site"), + "vm reporter should explain that framework.component requires a controlling direct \ + site: {err}", + ); + } +} diff --git a/runtime/mesh/src/dynamic_caps.rs b/runtime/mesh/src/dynamic_caps.rs index 0c4e0448..4651182e 100644 --- a/runtime/mesh/src/dynamic_caps.rs +++ b/runtime/mesh/src/dynamic_caps.rs @@ -16,9 +16,8 @@ pub const DYNAMIC_CAPS_REF_HOST: &str = "ref"; pub const DYNAMIC_CAPS_REF_VERSION: u32 = 1; pub const DYNAMIC_CAPS_API_URL_ENV: &str = "AMBER_DYNAMIC_CAPS_API_URL"; -pub const FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV: &str = "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL"; -pub const FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV: &str = - "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN"; +pub const FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME: &str = + "__amber_internal_site_controller"; pub const DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV: &str = "AMBER_DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/runtime/mesh/src/lib.rs b/runtime/mesh/src/lib.rs index 20144d0f..a9a818b1 100644 --- a/runtime/mesh/src/lib.rs +++ b/runtime/mesh/src/lib.rs @@ -207,9 +207,8 @@ pub struct MeshConfigTemplate { pub const MESH_PROVISION_PLAN_VERSION: &str = "2"; pub const MESH_CONFIG_FILENAME: &str = "mesh-config.json"; pub const MESH_IDENTITY_FILENAME: &str = "mesh-identity.json"; -pub const FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV: &str = "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL"; -pub const FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV: &str = - "AMBER_FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN"; +pub const FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME: &str = + dynamic_caps::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME; pub const DYNAMIC_CAPS_API_URL_ENV: &str = dynamic_caps::DYNAMIC_CAPS_API_URL_ENV; pub const DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV: &str = dynamic_caps::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV; diff --git a/runtime/router/src/control.rs b/runtime/router/src/control.rs index 90d12d34..c98ff7fc 100644 --- a/runtime/router/src/control.rs +++ b/runtime/router/src/control.rs @@ -1,3 +1,5 @@ +use std::env; + use super::*; #[derive(Clone)] @@ -428,17 +430,11 @@ pub(super) async fn handle_inbound( optional, } => match route.protocol { MeshProtocol::Http => { - let framework_route_id = (url_env - == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV) - .then(|| Arc::::from(route.route_id.as_str())); - let framework_peer_id = (url_env - == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV) - .then(|| Arc::::from(remote_id.as_str())); proxy_noise_to_external( &mut session, ExternalProxyRequest { - route_id: framework_route_id, - peer_id: framework_peer_id, + route_id: None, + peer_id: None, labels: HttpExchangeLabels::inbound_from_route( config.identity.id.clone().into(), remote_id.clone().into(), diff --git a/runtime/router/src/dynamic_caps.rs b/runtime/router/src/dynamic_caps.rs index d31cafd4..f5839d12 100644 --- a/runtime/router/src/dynamic_caps.rs +++ b/runtime/router/src/dynamic_caps.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, sync::Arc}; +use std::{collections::BTreeMap, env, sync::Arc}; use amber_mesh::{ MeshProtocol, @@ -34,7 +34,6 @@ const DYNAMIC_CAPS_WATCH_POLL_INTERVAL: Duration = Duration::from_millis(250); #[derive(Debug)] struct DynamicCapsControllerEnv { control_url: String, - control_auth_token: String, verify_key_raw: String, run_id: String, } @@ -111,7 +110,6 @@ pub(super) struct DynamicCapsRuntime { component_id: Arc, run_id: Arc, control_url: Arc, - control_auth_token: Arc, verify_key: ed25519_dalek::VerifyingKey, config: Arc, client: Arc, @@ -220,7 +218,7 @@ impl DynamicCapsRuntime { let Some(listen_addr) = config.dynamic_caps_listen else { return Ok(None); }; - let Some(control_env) = resolve_dynamic_caps_controller_env()? else { + let Some(control_env) = resolve_dynamic_caps_controller_env(config.as_ref())? else { tracing::warn!( target: "amber.internal", component_id = %config.identity.id, @@ -236,7 +234,6 @@ impl DynamicCapsRuntime { component_id: Arc::::from(format!("components.{}", config.identity.id)), run_id: Arc::::from(control_env.run_id), control_url: Arc::::from(control_env.control_url), - control_auth_token: Arc::::from(control_env.control_auth_token), verify_key, config, client, @@ -278,14 +275,11 @@ impl DynamicCapsRuntime { format!("failed to serialize dynamic capability control request: {err}"), ) })?; - let request = Request::builder() + let request_builder = Request::builder() .method(Method::POST) .uri(&url) - .header(header::CONTENT_TYPE, "application/json") - .header( - AMBER_FRAMEWORK_AUTH_HEADER, - self.control_auth_token.as_ref(), - ) + .header(header::CONTENT_TYPE, "application/json"); + let request = request_builder .body( Full::new(Bytes::from(request_body)) .map_err(|never| match never {}) @@ -1120,25 +1114,35 @@ fn required_dynamic_caps_env_var( }) } -fn resolve_dynamic_caps_controller_env() -> Result, RouterError> { - let control_url = nonempty_env_var(amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV); - let control_auth_token = - nonempty_env_var(amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV); +fn local_dynamic_caps_controller_url(config: &MeshConfig) -> Option { + config + .outbound + .iter() + .find(|route| { + route.protocol == MeshProtocol::Http + && route.capability + == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + }) + .map(|route| format!("http://127.0.0.1:{}", route.listen_port)) +} + +fn resolve_dynamic_caps_controller_env( + config: &MeshConfig, +) -> Result, RouterError> { + let local_control_url = local_dynamic_caps_controller_url(config); let verify_key_raw = nonempty_env_var(amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV); - if control_url.is_none() && control_auth_token.is_none() && verify_key_raw.is_none() { + if local_control_url.is_none() && verify_key_raw.is_none() { return Ok(None); } let run_id = required_dynamic_caps_env_var(SCENARIO_RUN_ID_ENV, nonempty_env_var(SCENARIO_RUN_ID_ENV))?; + let control_url = local_control_url.ok_or_else(|| { + RouterError::InvalidConfig( + "dynamic_caps_listen requires a local framework.component controller route".to_string(), + ) + })?; Ok(Some(DynamicCapsControllerEnv { - control_url: required_dynamic_caps_env_var( - amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, - control_url, - )?, - control_auth_token: required_dynamic_caps_env_var( - amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV, - control_auth_token, - )?, + control_url, verify_key_raw: required_dynamic_caps_env_var( amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, verify_key_raw, @@ -1160,7 +1164,7 @@ mod tests { } impl EnvGuard { - fn replace(pairs: [(&'static str, Option<&str>); 4]) -> Self { + fn replace(pairs: [(&'static str, Option<&str>); 2]) -> Self { let saved = pairs .iter() .map(|(name, value)| { @@ -1191,6 +1195,40 @@ mod tests { } } + fn mesh_config_with_internal_controller_route() -> MeshConfig { + MeshConfig { + identity: amber_mesh::MeshIdentity::generate("/component/test", None), + mesh_listen: "127.0.0.1:23000".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: Some("127.0.0.1:19000".parse().expect("dynamic caps listen")), + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: vec![OutboundRoute { + route_id: "component:/controller:__amber_internal_site_controller:http".to_string(), + rewrite_route_id: None, + slot: amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME.to_string(), + capability_kind: None, + capability_profile: None, + listen_port: 19001, + listen_addr: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + peer_addr: "controller:23000".to_string(), + peer_id: "/controller".to_string(), + capability: amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + .to_string(), + }], + transport: amber_mesh::TransportConfig::NoiseIk {}, + } + } + + fn mesh_config_without_internal_controller_route() -> MeshConfig { + let mut config = mesh_config_with_internal_controller_route(); + config.outbound.clear(); + config + } + #[test] fn resolve_dynamic_caps_controller_env_disables_listener_when_control_env_is_absent() { let _guard = ENV_LOCK @@ -1198,17 +1236,12 @@ mod tests { .lock() .expect("env lock"); let _env = EnvGuard::replace([ - (amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, None), - ( - amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV, - None, - ), (amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, None), (SCENARIO_RUN_ID_ENV, Some("run-1234")), ]); assert!( - resolve_dynamic_caps_controller_env() + resolve_dynamic_caps_controller_env(&mesh_config_without_internal_controller_route()) .expect("dynamic caps env should resolve") .is_none(), "sidecars without dynamic caps controller env should leave the listener disabled", @@ -1216,34 +1249,52 @@ mod tests { } #[test] - fn resolve_dynamic_caps_controller_env_rejects_partial_configuration() { + fn resolve_dynamic_caps_controller_env_rejects_missing_local_controller_route() { let _guard = ENV_LOCK .get_or_init(|| Mutex::new(())) .lock() .expect("env lock"); let _env = EnvGuard::replace([ ( - amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV, - Some("http://127.0.0.1:24000"), - ), - ( - amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV, - None, + amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, + Some("verify-key"), ), - (amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, None), (SCENARIO_RUN_ID_ENV, Some("run-1234")), ]); - let err = resolve_dynamic_caps_controller_env().expect_err("partial env must fail"); + let err = + resolve_dynamic_caps_controller_env(&mesh_config_without_internal_controller_route()) + .expect_err("dynamic caps without a local controller route must fail"); assert!( matches!( &err, RouterError::InvalidConfig(message) - if message.contains(amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV) + if message.contains("local framework.component controller route") ), - "partial dynamic caps controller env should fail with the missing variable name: {err}", + "missing local controller route should produce a targeted error: {err}", ); } + + #[test] + fn resolve_dynamic_caps_controller_env_prefers_local_controller_route_without_auth_token() { + let _guard = ENV_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("env lock"); + let _env = EnvGuard::replace([ + ( + amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, + Some("verify-key"), + ), + (SCENARIO_RUN_ID_ENV, Some("run-1234")), + ]); + + let config = mesh_config_with_internal_controller_route(); + let env = resolve_dynamic_caps_controller_env(&config) + .expect("dynamic caps env should resolve") + .expect("local controller route should enable dynamic caps"); + assert_eq!(env.control_url, "http://127.0.0.1:19001"); + } } #[derive(Clone)] diff --git a/runtime/router/src/external.rs b/runtime/router/src/external.rs index 3707582d..abf61a68 100644 --- a/runtime/router/src/external.rs +++ b/runtime/router/src/external.rs @@ -1,3 +1,5 @@ +use std::env; + use super::*; #[allow(clippy::result_large_err)] @@ -213,8 +215,7 @@ fn is_disallowed_external_ip(ip: IpAddr) -> bool { } fn allows_loopback_external_target(target: &ExternalTarget) -> bool { - target.url_env == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV - || (target.url_env.starts_with("AMBER_EXTERNAL_SLOT_") && target.url_env.ends_with("_URL")) + target.url_env.starts_with("AMBER_EXTERNAL_SLOT_") && target.url_env.ends_with("_URL") } pub(super) fn join_url(base: &Url, uri: &Uri) -> Url { diff --git a/runtime/router/src/lib.rs b/runtime/router/src/lib.rs index 5f5063dd..592b52dd 100644 --- a/runtime/router/src/lib.rs +++ b/runtime/router/src/lib.rs @@ -1,6 +1,5 @@ use std::{ collections::{BTreeMap, HashMap, HashSet, VecDeque}, - env, future::Future, io, net::{IpAddr, SocketAddr}, @@ -304,16 +303,6 @@ struct LocalHttpProxyState { const AMBER_ROUTE_ID_HEADER: &str = "x-amber-route-id"; const AMBER_PEER_ID_HEADER: &str = "x-amber-peer-id"; -const AMBER_FRAMEWORK_AUTH_HEADER: &str = "x-amber-framework-auth"; - -fn framework_component_auth_header_value() -> Option { - env::var(amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN_ENV) - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) - .and_then(|value| HeaderValue::from_str(&value).ok()) -} - #[derive(Clone)] struct OutboundHttpProxyState { upstream: Arc>>, @@ -1010,12 +999,6 @@ async fn proxy_local_http_request( HeaderValue::from_str(state.peer_id.as_ref()) .expect("peer id header value should be valid"), ); - if let Some(auth_token) = framework_component_auth_header_value() { - request_parts.headers.insert( - header::HeaderName::from_static(AMBER_FRAMEWORK_AUTH_HEADER), - auth_token, - ); - } if emit_telemetry { inject_trace_context(&span, &mut request_parts.headers); } @@ -1763,12 +1746,6 @@ async fn proxy_http_request(state: HttpProxyState, req: Request) -> Re .expect("peer id header value should be valid"), ); } - if let Some(auth_token) = framework_component_auth_header_value() { - parts.0.headers.insert( - header::HeaderName::from_static(AMBER_FRAMEWORK_AUTH_HEADER), - auth_token, - ); - } inject_trace_context(&span, &mut parts.0.headers); let request_body = capture_box_body( diff --git a/runtime/router/src/tests.rs b/runtime/router/src/tests.rs index 77d7b019..6fa61a26 100644 --- a/runtime/router/src/tests.rs +++ b/runtime/router/src/tests.rs @@ -66,6 +66,8 @@ fn empty_box_body() -> BoxBody { .boxed() } +const FRAMEWORK_AUTH_HEADER: &str = "x-amber-framework-auth"; + async fn response_text(response: Response) -> String { String::from_utf8( response @@ -1795,28 +1797,6 @@ fn resolve_http_external_target_with_override_rejects_loopback_ip_literals() { assert_eq!(response.status(), StatusCode::BAD_GATEWAY); } -#[test] -fn resolve_http_external_target_with_override_accepts_framework_loopback_ip_literals() { - let target = ExternalTarget { - name: "component".to_string(), - url_env: amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV.to_string(), - optional: false, - url_override: None, - }; - - let resolved = resolve_http_external_target_with_override( - &target, - Some("http://127.0.0.1:6167/base"), - &Uri::from_static("/v1/children"), - ) - .expect("framework target should resolve"); - - let ResolvedHttpExternalTarget::Http(url) = resolved else { - panic!("expected direct http target"); - }; - assert_eq!(url.as_str(), "http://127.0.0.1:6167/base/v1/children"); -} - #[test] fn resolve_http_external_target_with_override_accepts_external_slot_loopback_ip_literals() { let target = ExternalTarget { @@ -2281,6 +2261,100 @@ async fn outbound_http_proxy_overwrites_internal_framework_auth_headers() { .expect("upstream connection task should join"); } +#[tokio::test] +async fn outbound_http_proxy_does_not_attach_framework_auth_header_to_noise_hops() { + let captured = Arc::new(StdMutex::new(None::)); + let (upstream_client, upstream_server) = duplex(64 * 1024); + let captured_upstream = captured.clone(); + let upstream_task = tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let captured = captured_upstream.clone(); + async move { + *captured.lock().expect("capture lock") = req + .headers() + .get(FRAMEWORK_AUTH_HEADER) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + Ok::<_, std::convert::Infallible>( + Response::builder() + .status(StatusCode::OK) + .body(empty_box_body()) + .expect("upstream response should build"), + ) + } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(upstream_server), service) + .await + .expect("upstream server should complete"); + }); + + let (sender, conn) = client_http1::handshake(TokioIo::new(upstream_client)) + .await + .expect("upstream handshake should succeed"); + let upstream_conn_task = tokio::spawn(async move { + conn.await.expect("upstream connection should complete"); + }); + + let state = OutboundHttpProxyState { + upstream: Arc::new(Mutex::new(sender)), + plugins: Arc::from(Vec::>::new()), + route_id: Arc::::from("binding-route"), + peer_id: Arc::::from("/component/provider"), + labels: test_http_exchange_labels(), + dynamic_caps: None, + }; + + let (proxy_client, proxy_server) = duplex(64 * 1024); + let proxy_task = tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let state = state.clone(); + async move { + Ok::<_, std::convert::Infallible>(proxy_outbound_http_request(state, req).await) + } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(proxy_server), service) + .await + .expect("proxy server should complete"); + }); + + let (mut client, conn) = client_http1::handshake(TokioIo::new(proxy_client)) + .await + .expect("proxy handshake should succeed"); + let proxy_conn_task = tokio::spawn(async move { + conn.await.expect("proxy connection should complete"); + }); + + let response = client + .send_request( + Request::builder() + .uri("/binding") + .body(empty_box_body()) + .expect("request should build"), + ) + .await + .expect("proxy request should succeed"); + assert_eq!(response.status(), StatusCode::OK); + drop(response); + drop(client); + + assert_eq!( + *captured.lock().expect("capture lock"), + None, + "ordinary noise-to-noise binding hops must not carry the framework controller auth header", + ); + + proxy_task.await.expect("proxy task should join"); + proxy_conn_task + .await + .expect("proxy connection task should join"); + upstream_task.await.expect("upstream task should join"); + upstream_conn_task + .await + .expect("upstream connection task should join"); +} + #[tokio::test] async fn outbound_http_proxy_keeps_ephemeral_upstream_alive_for_response_body() { let (upstream_client, upstream_server) = duplex(64 * 1024); @@ -2491,7 +2565,106 @@ async fn proxy_noise_to_plain_preserves_connection_close_response_body() { } #[tokio::test] -async fn framework_external_proxy_preserves_forwarded_internal_auth_headers() { +async fn external_proxies_do_not_attach_framework_auth_headers() { + let captured = Arc::new(StdMutex::new(None::)); + let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) + .await + .expect("listener should bind"); + let addr = listener.local_addr().expect("listener addr"); + let captured_upstream = captured.clone(); + let upstream_task = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("upstream should accept"); + let service = service_fn(move |req: Request| { + let captured = captured_upstream.clone(); + async move { + *captured.lock().expect("capture lock") = req + .headers() + .get(FRAMEWORK_AUTH_HEADER) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + Ok::<_, std::convert::Infallible>( + Response::builder() + .status(StatusCode::OK) + .body(empty_box_body()) + .expect("upstream response should build"), + ) + } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await + .expect("upstream server should complete"); + }); + + let (client, vetted_external_addrs) = build_client(); + let external_overrides: ExternalOverrides = Arc::new(RwLock::new(HashMap::from([( + "catalog".to_string(), + format!("http://127.0.0.1:{}/", addr.port()), + )]))); + let state = HttpProxyState { + client, + target: ExternalTarget { + name: "catalog".to_string(), + url_env: "AMBER_EXTERNAL_SLOT_CATALOG_URL".to_string(), + optional: false, + url_override: None, + }, + labels: test_http_exchange_labels(), + config: Arc::new(test_mesh_config()), + external_overrides, + vetted_external_addrs, + mesh_upstream: Arc::new(Mutex::new(None)), + route_id: None, + peer_id: None, + }; + + let (proxy_client, proxy_server) = duplex(64 * 1024); + let proxy_task = tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let state = state.clone(); + async move { Ok::<_, std::convert::Infallible>(proxy_http_request(state, req).await) } + }); + http1::Builder::new() + .serve_connection(TokioIo::new(proxy_server), service) + .await + .expect("proxy server should complete"); + }); + + let (mut client, conn) = client_http1::handshake(TokioIo::new(proxy_client)) + .await + .expect("proxy handshake should succeed"); + let proxy_conn_task = tokio::spawn(async move { + conn.await.expect("proxy connection should complete"); + }); + + let response = client + .send_request( + Request::builder() + .uri("/catalog") + .body(empty_box_body()) + .expect("request should build"), + ) + .await + .expect("proxy request should succeed"); + assert_eq!(response.status(), StatusCode::OK); + drop(response); + drop(client); + + assert_eq!( + *captured.lock().expect("capture lock"), + None, + "ordinary external slots must not receive the framework auth header", + ); + + proxy_task.await.expect("proxy task should join"); + proxy_conn_task + .await + .expect("proxy connection task should join"); + upstream_task.await.expect("upstream task should join"); +} + +#[tokio::test] +async fn external_proxy_preserves_forwarded_internal_auth_headers() { let captured = Arc::new(StdMutex::new(None::<(String, String)>)); let listener = TcpListener::bind(SocketAddr::from(([127, 0, 0, 1], 0))) .await @@ -2532,14 +2705,14 @@ async fn framework_external_proxy_preserves_forwarded_internal_auth_headers() { let (client, vetted_external_addrs) = build_client(); let external_overrides: ExternalOverrides = Arc::new(RwLock::new(HashMap::from([( - "framework".to_string(), + "gateway".to_string(), format!("http://127.0.0.1:{}/", addr.port()), )]))); let state = HttpProxyState { client, target: ExternalTarget { - name: "framework".to_string(), - url_env: amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV.to_string(), + name: "gateway".to_string(), + url_env: "AMBER_EXTERNAL_SLOT_GATEWAY_URL".to_string(), optional: false, url_override: None, }, diff --git a/runtime/site-controller/src/api.rs b/runtime/site-controller/src/api.rs index d30a031b..3e1a0c4c 100644 --- a/runtime/site-controller/src/api.rs +++ b/runtime/site-controller/src/api.rs @@ -1,5 +1,6 @@ use super::{planner::*, state::*, *}; +#[cfg(test)] pub(crate) fn authorize_capability_instance<'a>( state: &'a FrameworkControlState, cap_instance_id: &str, @@ -15,6 +16,35 @@ pub(crate) fn authorize_capability_instance<'a>( Ok(record) } +pub(crate) fn authorize_capability_route<'a>( + state: &'a FrameworkControlState, + route_id: &str, + peer_id: &str, +) -> std::result::Result<&'a CapabilityInstanceRecord, ProtocolErrorResponse> { + let mut route_records = state + .capability_instances + .values() + .filter(|record| record.route_id == route_id); + let Some(first_record) = route_records.next() else { + return Err(protocol_error( + ProtocolErrorCode::Unauthorized, + "unknown framework capability instance", + )); + }; + if first_record.recipient_peer_id == peer_id { + return Ok(first_record); + } + route_records + .find(|record| record.recipient_peer_id == peer_id) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::Unauthorized, + "framework capability instance is not bound to the authenticated mesh peer", + ) + }) +} + +#[cfg(test)] pub(super) fn capability_instance_record<'a>( state: &'a FrameworkControlState, cap_instance_id: &str, @@ -404,7 +434,6 @@ pub(super) struct ResolvedTemplateBinding { #[derive(Clone)] pub(super) struct SyntheticSourceRecord { - pub(super) slot_name: String, pub(super) actual_source: BindingFrom, pub(super) source_child_id: Option, pub(super) weak: bool, diff --git a/runtime/site-controller/src/control_state_mcp.rs b/runtime/site-controller/src/control_state_mcp.rs index ba21d733..54ed6d84 100644 --- a/runtime/site-controller/src/control_state_mcp.rs +++ b/runtime/site-controller/src/control_state_mcp.rs @@ -63,7 +63,7 @@ impl FrameworkDynamicCapsMcp { .extensions .get::() .ok_or_else(|| McpError::invalid_request("missing HTTP request context", None))?; - super::http::authorize_framework_auth_header( + super::http::authorize_control_state_auth_header( &parts.headers, self.app.control.control_state_auth_token.as_ref(), ) diff --git a/runtime/site-controller/src/default_runtime.rs b/runtime/site-controller/src/default_runtime.rs index 4e27e671..ad1a5ea8 100644 --- a/runtime/site-controller/src/default_runtime.rs +++ b/runtime/site-controller/src/default_runtime.rs @@ -158,8 +158,6 @@ struct SiteManagerState { #[serde(default, skip_serializing_if = "Option::is_none")] router_public_key_b64: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - site_controller_pid: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] site_controller_url: Option, #[serde(default, skip_serializing_if = "Option::is_none")] last_error: Option, @@ -196,8 +194,6 @@ struct SiteSupervisorPlan { #[serde(default, skip_serializing_if = "Option::is_none")] observability_endpoint: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - site_controller_plan_path: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] site_controller_url: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] launch_env: BTreeMap, @@ -1889,7 +1885,6 @@ mod tests { kubernetes_consumer_router_mesh_addr: None, router_identity_id: None, router_public_key_b64: None, - site_controller_pid: None, site_controller_url: None, last_error: None, }; @@ -1950,7 +1945,6 @@ mod tests { kubernetes_consumer_router_mesh_addr: None, router_identity_id: None, router_public_key_b64: None, - site_controller_pid: None, site_controller_url: None, last_error: None, }; diff --git a/runtime/site-controller/src/default_runtime/compose_controller.rs b/runtime/site-controller/src/default_runtime/compose_controller.rs index 343460ad..922384f5 100644 --- a/runtime/site-controller/src/default_runtime/compose_controller.rs +++ b/runtime/site-controller/src/default_runtime/compose_controller.rs @@ -4,34 +4,66 @@ use serde_json::json; use super::*; -const COMPOSE_MESH_NETWORK_NAME: &str = "amber_mesh"; const COMPOSE_PROVISIONER_SERVICE_NAME: &str = "amber-provisioner"; const COMPOSE_ROUTER_SERVICE_NAME: &str = "amber-router"; const COMPOSE_ROUTER_CONTROL_INIT_SERVICE_NAME: &str = "amber-router-control-init"; const COMPOSE_ROUTER_CONTROL_SOCKET_DIR: &str = "/amber/control"; const COMPOSE_ROUTER_CONTROL_VOLUME_NAME: &str = "amber-router-control"; const DOCKER_SOCK_PATH: &str = "/var/run/docker.sock"; +const COMPOSE_CONTROLLER_PLAN_PATH: &str = "/amber/site/state/site-controller-plan.json"; -fn compose_site_controller_env(plan: &SiteControllerPlan) -> serde_json::Value { - serde_json::Value::Object( - plan.launch_env - .iter() - .map(|(key, value)| (key.clone(), serde_json::Value::String(value.clone()))) - .collect(), - ) +fn ensure_string_sequence_contains(sequence: &mut serde_yaml::Sequence, value: &str) { + if sequence + .iter() + .any(|entry| entry.as_str().is_some_and(|existing| existing == value)) + { + return; + } + sequence.push(yaml_string(value)); } -pub fn inject_compose_site_controller( - artifact_root: &Path, +fn ensure_controller_environment( + service: &mut serde_yaml::Mapping, plan: &SiteControllerPlan, - plan_path: &Path, - controller_image: &str, ) -> Result<()> { - let compose_path = artifact_root.join("compose.yaml"); - let mut document = read_compose_document(&compose_path)?; - let services = compose_services_mut(&mut document, &compose_path)?; + if plan.launch_env.is_empty() { + return Ok(()); + } + match service + .entry(yaml_string("environment")) + .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) + { + serde_yaml::Value::Sequence(sequence) => { + for (key, value) in &plan.launch_env { + let entry = format!("{key}={value}"); + let already_present = sequence.iter().any(|existing| { + existing.as_str().is_some_and(|existing| { + existing == entry || existing.starts_with(&format!("{key}=")) + }) + }); + if !already_present { + sequence.push(yaml_string(&entry)); + } + } + Ok(()) + } + serde_yaml::Value::Mapping(mapping) => { + for (key, value) in &plan.launch_env { + mapping.insert(yaml_string(key), yaml_string(value)); + } + Ok(()) + } + _ => Err(miette::miette!( + "compose site controller service has an unsupported environment shape" + )), + } +} - let mut depends_on = serde_yaml::Mapping::new(); +fn ensure_controller_depends_on( + service: &mut serde_yaml::Mapping, + available_services: &serde_yaml::Mapping, +) -> Result<()> { + let mut desired = serde_yaml::Mapping::new(); for (service_name, condition) in [ ( COMPOSE_PROVISIONER_SERVICE_NAME, @@ -43,8 +75,8 @@ pub fn inject_compose_site_controller( ), (COMPOSE_ROUTER_SERVICE_NAME, "service_started"), ] { - if services.contains_key(yaml_string(service_name)) { - depends_on.insert( + if available_services.contains_key(yaml_string(service_name)) { + desired.insert( yaml_string(service_name), serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( yaml_string("condition"), @@ -53,47 +85,118 @@ pub fn inject_compose_site_controller( ); } } + if desired.is_empty() { + return Ok(()); + } + + let current = service + .entry(yaml_string("depends_on")) + .or_insert_with(|| serde_yaml::Value::Mapping(serde_yaml::Mapping::new())); + match current { + serde_yaml::Value::Mapping(mapping) => { + for (key, value) in desired { + mapping.insert(key, value); + } + Ok(()) + } + serde_yaml::Value::Sequence(sequence) => { + let mut mapping = serde_yaml::Mapping::new(); + for value in sequence.iter() { + let Some(name) = value.as_str() else { + return Err(miette::miette!( + "compose site controller service depends_on sequence contains a \ + non-string entry" + )); + }; + mapping.insert( + yaml_string(name), + serde_yaml::Value::Mapping(serde_yaml::Mapping::from_iter([( + yaml_string("condition"), + yaml_string("service_started"), + )])), + ); + } + for (key, value) in desired { + mapping.insert(key, value); + } + *current = serde_yaml::Value::Mapping(mapping); + Ok(()) + } + _ => Err(miette::miette!( + "compose site controller service has an unsupported depends_on shape" + )), + } +} - let networks = std::collections::BTreeMap::from([( - COMPOSE_MESH_NETWORK_NAME.to_string(), - serde_json::Value::Object(serde_json::Map::new()), - )]); - let service = json!({ - "image": controller_image, - // The controller must be able to reach both the router-control volume and the mounted - // run root. The router-control init service locks `/amber/control` down to `0700`, so a - // host-derived UID/GID breaks Linux compose startup. - "user": "0:0", - "command": ["--plan", plan_path.display().to_string()], - "environment": compose_site_controller_env(plan), - "networks": networks, - "extra_hosts": ["host.docker.internal:host-gateway"], - "healthcheck": { +pub fn inject_compose_site_controller( + artifact_root: &Path, + plan: &SiteControllerPlan, + plan_path: &Path, + controller_image: &str, +) -> Result<()> { + let compose_path = artifact_root.join("compose.yaml"); + let mut document = read_compose_document(&compose_path)?; + let services = compose_services_mut(&mut document, &compose_path)?; + let available_services = services.clone(); + let service = services + .get_mut(yaml_string(SITE_CONTROLLER_SERVICE_NAME)) + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or_else(|| { + miette::miette!( + "compose file {} is missing services.{SITE_CONTROLLER_SERVICE_NAME}", + compose_path.display() + ) + })?; + + service.insert(yaml_string("image"), yaml_string(controller_image)); + service.insert(yaml_string("user"), yaml_string("0:0")); + service.insert( + yaml_string("healthcheck"), + serde_yaml::to_value(json!({ "test": [ "CMD-SHELL", - "wget -qO- http://127.0.0.1:4100/healthz | grep -q '\"ok\":true'" + format!( + "wget -qO- http://127.0.0.1:{}/healthz | grep -q '\"ok\":true'", + plan.listen_addr.port() + ) ], "interval": "2s", "timeout": "2s", "retries": 30, "start_period": "1s" - }, - "volumes": [ - format!("{}:{}", plan.run_root, plan.run_root), - format!( - "{COMPOSE_ROUTER_CONTROL_VOLUME_NAME}:{COMPOSE_ROUTER_CONTROL_SOCKET_DIR}" - ), - format!("{DOCKER_SOCK_PATH}:{DOCKER_SOCK_PATH}") - ], - "depends_on": depends_on, - "restart": "unless-stopped" - }); - services.insert( - yaml_string(SITE_CONTROLLER_SERVICE_NAME), - serde_yaml::to_value(service) - .into_diagnostic() - .wrap_err("failed to serialize compose site controller service")?, + })) + .into_diagnostic() + .wrap_err("failed to serialize compose site controller healthcheck")?, ); + service.insert(yaml_string("restart"), yaml_string("unless-stopped")); + + let extra_hosts = service + .entry(yaml_string("extra_hosts")) + .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) + .as_sequence_mut() + .ok_or_else(|| { + miette::miette!("compose site controller service has a non-sequence extra_hosts field") + })?; + ensure_string_sequence_contains(extra_hosts, "host.docker.internal:host-gateway"); + + let volumes = service + .entry(yaml_string("volumes")) + .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) + .as_sequence_mut() + .ok_or_else(|| { + miette::miette!("compose site controller service has a non-sequence volumes field") + })?; + for volume in [ + format!("{}:{}", plan.run_root, plan.run_root), + format!("{}:{COMPOSE_CONTROLLER_PLAN_PATH}", plan_path.display()), + format!("{COMPOSE_ROUTER_CONTROL_VOLUME_NAME}:{COMPOSE_ROUTER_CONTROL_SOCKET_DIR}"), + format!("{DOCKER_SOCK_PATH}:{DOCKER_SOCK_PATH}"), + ] { + ensure_string_sequence_contains(volumes, &volume); + } + + ensure_controller_environment(service, plan)?; + ensure_controller_depends_on(service, &available_services)?; let rendered = serde_yaml::to_string(&document) .into_diagnostic() @@ -134,7 +237,7 @@ mod tests { state_root: run_root.join("state-root").display().to_string(), site_state_root: run_root.join("site-state").display().to_string(), artifact_dir: run_root.join("artifact").display().to_string(), - auth_token: "token".to_string(), + control_state_auth_token: "token".to_string(), dynamic_caps_token_verify_key_b64: "verify".to_string(), storage_root: None, runtime_root: None, @@ -156,7 +259,7 @@ mod tests { let compose_path = temp.path().join("compose.yaml"); fs::write( &compose_path, - "services:\n amber-router:\n image: ghcr.io/rdi-foundation/amber-router:test\n", + "services:\n amber-router:\n image: ghcr.io/rdi-foundation/amber-router:test\n amber-site-controller:\n image: __amber_internal/site-controller\n environment:\n - AMBER_DYNAMIC_CAPS_API_URL=http://127.0.0.1:19000\n", ) .expect("compose file should write"); @@ -182,13 +285,34 @@ mod tests { .expect("site controller service should exist"); let environment = service .get(yaml_string("environment")) - .and_then(serde_yaml::Value::as_mapping) - .expect("site controller service should include environment"); - assert_eq!( + .and_then(serde_yaml::Value::as_sequence) + .expect("site controller service should include environment entries"); + assert!( environment - .get(yaml_string("AMBER_DEV_IMAGE_TAGS")) - .and_then(serde_yaml::Value::as_str), - Some("router=dev-tag,helper=dev-tag"), + .iter() + .any(|value| value.as_str().is_some_and(|value| { + value == "AMBER_DEV_IMAGE_TAGS=router=dev-tag,helper=dev-tag" + })), + "launch env should be merged into the existing environment list: {service:?}" + ); + assert!( + environment + .iter() + .any(|value| value.as_str().is_some_and(|value| { + value == "AMBER_DYNAMIC_CAPS_API_URL=http://127.0.0.1:19000" + })), + "existing controller program env must be preserved: {service:?}" + ); + let volumes = service + .get(yaml_string("volumes")) + .and_then(serde_yaml::Value::as_sequence) + .expect("site controller service should include volumes"); + assert!( + volumes.iter().any(|value| value + .as_str() + .is_some_and(|value| { value == format!("{}:{}", plan.run_root, plan.run_root) })), + "compose controller bootstrap should mount the run root into the existing service: \ + {service:?}" ); } } diff --git a/runtime/site-controller/src/default_runtime/dynamic_routes.rs b/runtime/site-controller/src/default_runtime/dynamic_routes.rs index 1e949e19..134126dc 100644 --- a/runtime/site-controller/src/default_runtime/dynamic_routes.rs +++ b/runtime/site-controller/src/default_runtime/dynamic_routes.rs @@ -1599,7 +1599,41 @@ mod direct_input_tests { version: amber_mesh::MESH_PROVISION_PLAN_VERSION.to_string(), identity_seed: None, existing_peer_identities: Vec::new(), - targets: vec![MeshProvisionTarget { + targets: vec![ + MeshProvisionTarget { + kind: MeshProvisionTargetKind::Component, + config: MeshConfigTemplate { + identity: amber_mesh::MeshIdentityTemplate { + id: "/site/local/controller".to_string(), + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:23001".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: vec![InboundRoute { + route_id: amber_mesh::component_route_id( + "/site/local/controller", + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + MeshProtocol::Http, + ), + capability: amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME.to_string(), + capability_kind: None, + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: InboundTarget::Local { port: 8080 }, + allowed_issuers: vec!["/site/local/router".to_string()], + }], + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + output: MeshProvisionOutput::Filesystem { + dir: "mesh/controller".to_string(), + }, + }, + MeshProvisionTarget { kind: MeshProvisionTargetKind::Router, config: MeshConfigTemplate { identity: amber_mesh::MeshIdentityTemplate { @@ -1618,7 +1652,8 @@ mod direct_input_tests { output: MeshProvisionOutput::Filesystem { dir: "mesh/router".to_string(), }, - }], + } + ], }, ) .expect("write mesh plan"); @@ -1642,7 +1677,11 @@ mod direct_input_tests { .expect("inject routes"); let plan: MeshProvisionPlan = read_json(&path, "mesh provision plan").expect("plan"); - let router = &plan.targets[0]; + let router = plan + .targets + .iter() + .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) + .expect("router target should remain present"); assert!( router.config.inbound.iter().any(|route| { route.route_id == site_controller_internal_route_id("local") @@ -1650,9 +1689,26 @@ mod direct_input_tests { && route.allowed_issuers == vec!["/site/peer/router".to_string()] && matches!( route.target, - InboundTarget::External { ref url_env, optional } - if url_env == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV - && !optional + InboundTarget::MeshForward { + ref peer_id, + ref peer_addr, + ref route_id, + ref capability, + } if peer_id == "/site/local/controller" + && peer_addr + == if cfg!(target_os = "linux") { + "10.0.2.2:23001" + } else { + "127.0.0.1:23001" + } + && route_id + == &amber_mesh::component_route_id( + "/site/local/controller", + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + MeshProtocol::Http, + ) + && capability + == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME ) }), "router should expose a local inbound route for peer site-controller traffic", diff --git a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs index e018807f..159a37d8 100644 --- a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs +++ b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs @@ -24,6 +24,7 @@ const KUBERNETES_CONTROLLER_ROLE_PATH: &str = "02-rbac/amber-site-controller-rol const KUBERNETES_CONTROLLER_ROLE_BINDING_PATH: &str = "02-rbac/amber-site-controller-rolebinding.yaml"; const KUBERNETES_CONTROLLER_DEPLOYMENT_PATH: &str = "03-deployments/amber-site-controller.yaml"; +#[cfg(test)] const KUBERNETES_CONTROLLER_SERVICE_PATH: &str = "04-services/amber-site-controller.yaml"; const KUBERNETES_CONTROLLER_SEED_VOLUME: &str = "controller-seed"; const KUBERNETES_CONTROLLER_STATE_VOLUME: &str = "controller-state"; @@ -35,13 +36,6 @@ const KUBERNETES_CONTROLLER_STATE_PATH: &str = "/amber/site/state/site-controlle const KUBERNETES_CONTROLLER_DESIRED_LINKS_PATH: &str = "/amber/site/state/desired-links.json"; const KUBERNETES_CONTROLLER_SEED_ROOT: &str = "/amber/seed"; -fn kubernetes_env_entries(plan: &SiteControllerPlan) -> Vec { - plan.launch_env - .iter() - .map(|(name, value)| json!({ "name": name, "value": value })) - .collect() -} - pub fn inject_kubernetes_site_controller( artifact_root: &Path, plan: &SiteControllerPlan, @@ -51,8 +45,6 @@ pub fn inject_kubernetes_site_controller( let seed_configmap = build_kubernetes_controller_seed_configmap(artifact_root, plan, &embedded_plan)?; let labels = kubernetes_controller_labels(); - let selector = kubernetes_controller_selector(); - let controller_env = kubernetes_env_entries(plan); write_yaml_artifact( artifact_root.join(KUBERNETES_CONTROLLER_SEED_CONFIGMAP_PATH), @@ -133,110 +125,10 @@ pub fn inject_kubernetes_site_controller( } }), )?; - write_yaml_artifact( + patch_kubernetes_controller_deployment( artifact_root.join(KUBERNETES_CONTROLLER_DEPLOYMENT_PATH), - &json!({ - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "name": SITE_CONTROLLER_SERVICE_NAME, - "labels": labels, - }, - "spec": { - "replicas": 1, - "selector": { - "matchLabels": selector, - }, - "template": { - "metadata": { - "labels": labels, - }, - "spec": { - "serviceAccountName": SITE_CONTROLLER_SERVICE_NAME, - "automountServiceAccountToken": true, - "initContainers": [{ - "name": "seed-site-controller", - "image": controller_image, - "command": [ - "sh", - "-lc", - format!( - "set -eu\nmkdir -p {KUBERNETES_CONTROLLER_STATE_ROOT} \ - {KUBERNETES_CONTROLLER_ARTIFACT_ROOT}\ncp \ - {KUBERNETES_CONTROLLER_SEED_ROOT}/site-controller-plan.json \ - {KUBERNETES_CONTROLLER_PLAN_PATH}\ncp \ - {KUBERNETES_CONTROLLER_SEED_ROOT}/site-controller-state.json \ - {KUBERNETES_CONTROLLER_STATE_PATH}\ncp \ - {KUBERNETES_CONTROLLER_SEED_ROOT}/desired-links.json \ - {KUBERNETES_CONTROLLER_DESIRED_LINKS_PATH}\nbase64 -d \ - {KUBERNETES_CONTROLLER_SEED_ROOT}/artifact.tar.b64 | tar \ - -xf - -C {KUBERNETES_CONTROLLER_ARTIFACT_ROOT}\n" - ) - ], - "volumeMounts": [ - { - "name": KUBERNETES_CONTROLLER_SEED_VOLUME, - "mountPath": KUBERNETES_CONTROLLER_SEED_ROOT, - "readOnly": true, - }, - { - "name": KUBERNETES_CONTROLLER_STATE_VOLUME, - "mountPath": KUBERNETES_CONTROLLER_SITE_ROOT, - } - ] - }], - "containers": [{ - "name": SITE_CONTROLLER_SERVICE_NAME, - "image": controller_image, - "args": ["--plan", KUBERNETES_CONTROLLER_PLAN_PATH], - "env": controller_env, - "ports": [{ - "name": "http", - "containerPort": SITE_CONTROLLER_PORT, - "protocol": "TCP", - }], - "volumeMounts": [{ - "name": KUBERNETES_CONTROLLER_STATE_VOLUME, - "mountPath": KUBERNETES_CONTROLLER_SITE_ROOT, - }] - }], - "volumes": [ - { - "name": KUBERNETES_CONTROLLER_SEED_VOLUME, - "configMap": { - "name": format!("{SITE_CONTROLLER_SERVICE_NAME}-seed"), - } - }, - { - "name": KUBERNETES_CONTROLLER_STATE_VOLUME, - "emptyDir": {} - } - ] - } - } - } - }), - )?; - write_yaml_artifact( - artifact_root.join(KUBERNETES_CONTROLLER_SERVICE_PATH), - &json!({ - "apiVersion": "v1", - "kind": "Service", - "metadata": { - "name": SITE_CONTROLLER_SERVICE_NAME, - "labels": labels, - }, - "spec": { - "selector": selector, - "ports": [{ - "name": "http", - "port": SITE_CONTROLLER_PORT, - "targetPort": SITE_CONTROLLER_PORT, - "protocol": "TCP", - }], - "type": "ClusterIP", - } - }), + plan, + controller_image, )?; add_kubernetes_resource_paths( artifact_root, @@ -245,8 +137,6 @@ pub fn inject_kubernetes_site_controller( KUBERNETES_CONTROLLER_SERVICE_ACCOUNT_PATH, KUBERNETES_CONTROLLER_ROLE_PATH, KUBERNETES_CONTROLLER_ROLE_BINDING_PATH, - KUBERNETES_CONTROLLER_DEPLOYMENT_PATH, - KUBERNETES_CONTROLLER_SERVICE_PATH, ], )?; ensure_kubernetes_router_allows_site_controller_ingress( @@ -255,11 +145,226 @@ pub fn inject_kubernetes_site_controller( ) } +fn upsert_named_sequence_entry( + sequence: &mut serde_yaml::Sequence, + name: &str, + value: serde_yaml::Value, +) { + if let Some(existing) = sequence.iter_mut().find(|entry| { + entry + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|existing| existing == name) + }) { + *existing = value; + } else { + sequence.push(value); + } +} + +fn merge_kubernetes_controller_env( + container: &mut serde_yaml::Mapping, + plan: &SiteControllerPlan, +) -> Result<()> { + if plan.launch_env.is_empty() { + return Ok(()); + } + let env = container + .entry(yaml_string("env")) + .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) + .as_sequence_mut() + .ok_or_else(|| miette::miette!("controller deployment env must be a sequence"))?; + for (name, value) in &plan.launch_env { + upsert_named_sequence_entry( + env, + name, + serde_yaml::to_value(json!({ "name": name, "value": value })) + .into_diagnostic() + .wrap_err("failed to serialize controller env entry")?, + ); + } + Ok(()) +} + +fn ensure_kubernetes_volume_mount( + container: &mut serde_yaml::Mapping, + name: &str, + mount_path: &str, + read_only: Option, +) -> Result<()> { + let mounts = container + .entry(yaml_string("volumeMounts")) + .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) + .as_sequence_mut() + .ok_or_else(|| miette::miette!("controller deployment volumeMounts must be a sequence"))?; + let mut mount = json!({ + "name": name, + "mountPath": mount_path, + }); + if let Some(read_only) = read_only { + mount["readOnly"] = serde_json::Value::Bool(read_only); + } + upsert_named_sequence_entry( + mounts, + name, + serde_yaml::to_value(mount) + .into_diagnostic() + .wrap_err("failed to serialize controller volume mount")?, + ); + Ok(()) +} + +fn patch_kubernetes_controller_deployment( + path: PathBuf, + plan: &SiteControllerPlan, + controller_image: &str, +) -> Result<()> { + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut deployment: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid {}", path.display()))?; + let pod_spec = deployment + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("spec"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .and_then(|spec| spec.get_mut(yaml_string("template"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .and_then(|template| template.get_mut(yaml_string("spec"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or_else(|| { + miette::miette!( + "controller deployment {} is missing spec.template.spec", + path.display() + ) + })?; + pod_spec.insert( + yaml_string("serviceAccountName"), + yaml_string(SITE_CONTROLLER_SERVICE_NAME), + ); + pod_spec.insert( + yaml_string("automountServiceAccountToken"), + serde_yaml::Value::Bool(true), + ); + + let init_containers = pod_spec + .entry(yaml_string("initContainers")) + .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) + .as_sequence_mut() + .ok_or_else(|| { + miette::miette!("controller deployment initContainers must be a sequence") + })?; + upsert_named_sequence_entry( + init_containers, + "seed-site-controller", + serde_yaml::to_value(json!({ + "name": "seed-site-controller", + "image": controller_image, + "command": [ + "sh", + "-lc", + format!( + "set -eu + mkdir -p {KUBERNETES_CONTROLLER_STATE_ROOT} \ + {KUBERNETES_CONTROLLER_ARTIFACT_ROOT} +cp \ + {KUBERNETES_CONTROLLER_SEED_ROOT}/site-controller-plan.json \ + {KUBERNETES_CONTROLLER_PLAN_PATH} +cp \ + {KUBERNETES_CONTROLLER_SEED_ROOT}/site-controller-state.json \ + {KUBERNETES_CONTROLLER_STATE_PATH} +cp \ + {KUBERNETES_CONTROLLER_SEED_ROOT}/desired-links.json \ + {KUBERNETES_CONTROLLER_DESIRED_LINKS_PATH} +base64 -d \ + {KUBERNETES_CONTROLLER_SEED_ROOT}/artifact.tar.b64 | tar \ + -xf - -C {KUBERNETES_CONTROLLER_ARTIFACT_ROOT} +" + ) + ], + "volumeMounts": [ + { + "name": KUBERNETES_CONTROLLER_SEED_VOLUME, + "mountPath": KUBERNETES_CONTROLLER_SEED_ROOT, + "readOnly": true, + }, + { + "name": KUBERNETES_CONTROLLER_STATE_VOLUME, + "mountPath": KUBERNETES_CONTROLLER_SITE_ROOT, + } + ] + })) + .into_diagnostic() + .wrap_err("failed to serialize controller seed init container")?, + ); + + let containers = pod_spec + .get_mut(yaml_string("containers")) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!("controller deployment is missing spec.template.spec.containers") + })?; + let main_index = containers + .iter() + .position(|container| { + container + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|name| name == "main") + }) + .unwrap_or(0); + let main = containers + .get_mut(main_index) + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or_else(|| miette::miette!("controller deployment has no mutable main container"))?; + main.insert(yaml_string("image"), yaml_string(controller_image)); + merge_kubernetes_controller_env(main, plan)?; + ensure_kubernetes_volume_mount( + main, + KUBERNETES_CONTROLLER_STATE_VOLUME, + KUBERNETES_CONTROLLER_SITE_ROOT, + None, + )?; + + let volumes = pod_spec + .entry(yaml_string("volumes")) + .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) + .as_sequence_mut() + .ok_or_else(|| miette::miette!("controller deployment volumes must be a sequence"))?; + upsert_named_sequence_entry( + volumes, + KUBERNETES_CONTROLLER_SEED_VOLUME, + serde_yaml::to_value(json!({ + "name": KUBERNETES_CONTROLLER_SEED_VOLUME, + "configMap": { + "name": format!("{SITE_CONTROLLER_SERVICE_NAME}-seed"), + } + })) + .into_diagnostic() + .wrap_err("failed to serialize controller seed volume")?, + ); + upsert_named_sequence_entry( + volumes, + KUBERNETES_CONTROLLER_STATE_VOLUME, + serde_yaml::to_value(json!({ + "name": KUBERNETES_CONTROLLER_STATE_VOLUME, + "emptyDir": {} + })) + .into_diagnostic() + .wrap_err("failed to serialize controller state volume")?, + ); + + write_yaml_artifact(path, &deployment) +} + fn build_embedded_kubernetes_controller_plan(plan: &SiteControllerPlan) -> SiteControllerPlan { let mut embedded = plan.clone(); - embedded.listen_addr = SocketAddr::from(([0, 0, 0, 0], SITE_CONTROLLER_PORT)); - embedded.authority_url = - format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{SITE_CONTROLLER_PORT}"); + let port = plan.listen_addr.port(); + embedded.listen_addr = SocketAddr::from(([0, 0, 0, 0], port)); + embedded.authority_url = format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{port}"); embedded.local_router_control = Some(format!( "{KUBERNETES_ROUTER_NAME}:{KUBERNETES_ROUTER_CONTROL_PORT}" )); @@ -551,7 +656,7 @@ fn kubernetes_controller_selector() -> BTreeMap { )]) } -fn write_yaml_artifact(path: PathBuf, value: &serde_json::Value) -> Result<()> { +fn write_yaml_artifact(path: PathBuf, value: &impl serde::Serialize) -> Result<()> { let parent = path.parent().ok_or_else(|| { miette::miette!("artifact path {} has no parent directory", path.display()) })?; @@ -578,6 +683,7 @@ mod tests { use super::*; fn test_plan(root: &Path) -> SiteControllerPlan { + let controller_port = 32123; SiteControllerPlan { schema: "amber.framework_component.site_controller_plan".to_string(), version: 1, @@ -585,8 +691,8 @@ mod tests { mesh_scope: "scope".to_string(), site_id: "kind_local".to_string(), kind: SiteKind::Kubernetes, - listen_addr: SocketAddr::from(([127, 0, 0, 1], 4100)), - authority_url: "http://amber-site-controller:4100".to_string(), + listen_addr: SocketAddr::from(([127, 0, 0, 1], controller_port)), + authority_url: format!("http://amber-site-controller:{controller_port}"), router_identity_id: "/site/kind_local/router".to_string(), peer_site_router_urls: BTreeMap::new(), peer_router_identities: BTreeMap::new(), @@ -600,7 +706,7 @@ mod tests { state_root: root.join("state-root").display().to_string(), site_state_root: root.join("site-state").display().to_string(), artifact_dir: root.join("artifact").display().to_string(), - auth_token: "token".to_string(), + control_state_auth_token: "token".to_string(), dynamic_caps_token_verify_key_b64: "verify".to_string(), storage_root: None, runtime_root: None, @@ -624,7 +730,8 @@ mod tests { fs::create_dir_all(artifact_root.join("05-networkpolicies")).expect("netpol dir"); fs::write( artifact_root.join("kustomization.yaml"), - "resources:\n - 04-services/amber-router.yaml\n - \ + "resources:\n - 03-deployments/amber-site-controller.yaml\n - \ + 04-services/amber-site-controller.yaml\n - 04-services/amber-router.yaml\n - \ 05-networkpolicies/amber-router-netpol.yaml\n", ) .expect("kustomization should write"); @@ -640,6 +747,47 @@ mod tests { amber-router-netpol\nspec:\n ingress: []\n", ) .expect("router netpol should write"); + fs::create_dir_all(artifact_root.join("03-deployments")).expect("deployments dir"); + fs::write( + artifact_root.join(KUBERNETES_CONTROLLER_DEPLOYMENT_PATH), + r#" +apiVersion: apps/v1 +kind: Deployment +metadata: + name: amber-site-controller +spec: + template: + spec: + containers: + - name: main + image: __amber_internal/site-controller + args: + - --plan + - /amber/site/state/site-controller-plan.json + env: + - name: EXISTING_ENV + value: kept + - name: sidecar + image: ghcr.io/rdi-foundation/amber-router:test +"#, + ) + .expect("controller deployment should write"); + fs::write( + artifact_root.join(KUBERNETES_CONTROLLER_SERVICE_PATH), + r#" +apiVersion: v1 +kind: Service +metadata: + name: amber-site-controller +spec: + ports: + - name: framework-component + port: 32123 + targetPort: 32123 + protocol: TCP +"#, + ) + .expect("controller service should write"); let plan = test_plan(artifact_root); fs::write(&plan.state_path, "{}").expect("state should write"); @@ -662,7 +810,7 @@ mod tests { .expect("controller deployment should exist"); let deployment: serde_yaml::Value = serde_yaml::from_str(&deployment_raw).expect("deployment yaml should parse"); - let env = deployment + let containers = deployment .as_mapping() .and_then(|root| root.get(yaml_string("spec"))) .and_then(serde_yaml::Value::as_mapping) @@ -672,11 +820,27 @@ mod tests { .and_then(serde_yaml::Value::as_mapping) .and_then(|spec| spec.get(yaml_string("containers"))) .and_then(serde_yaml::Value::as_sequence) - .and_then(|containers| containers.first()) + .expect("controller deployment should include containers"); + assert_eq!( + containers.len(), + 2, + "bootstrap should patch the rendered controller deployment instead of replacing it" + ); + let main = containers + .iter() + .find(|container| { + container + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|name| name == "main") + }) .and_then(serde_yaml::Value::as_mapping) - .and_then(|container| container.get(yaml_string("env"))) + .expect("controller deployment should keep the main container"); + let env = main + .get(yaml_string("env")) .and_then(serde_yaml::Value::as_sequence) - .expect("controller container should include env"); + .expect("controller main container should include env"); assert!(env.iter().any(|entry| { entry.as_mapping().is_some_and(|mapping| { mapping.get(yaml_string("name")) == Some(&yaml_string("AMBER_DEV_IMAGE_TAGS")) @@ -684,5 +848,11 @@ mod tests { == Some(&yaml_string("router=dev-tag,helper=dev-tag")) }) })); + assert!(env.iter().any(|entry| { + entry.as_mapping().is_some_and(|mapping| { + mapping.get(yaml_string("name")) == Some(&yaml_string("EXISTING_ENV")) + && mapping.get(yaml_string("value")) == Some(&yaml_string("kept")) + }) + })); } } diff --git a/runtime/site-controller/src/default_runtime/site_artifacts.rs b/runtime/site-controller/src/default_runtime/site_artifacts.rs index cb5cf6ae..437b6944 100644 --- a/runtime/site-controller/src/default_runtime/site_artifacts.rs +++ b/runtime/site-controller/src/default_runtime/site_artifacts.rs @@ -66,6 +66,52 @@ fn local_router_mesh_peer_addr(mesh_port: u16) -> String { std::net::SocketAddr::from(([127, 0, 0, 1], mesh_port)).to_string() } +fn site_controller_component_target(plan: &MeshProvisionPlan) -> Result<&MeshProvisionTarget> { + plan.targets + .iter() + .find(|target| { + matches!(target.kind, MeshProvisionTargetKind::Component) + && target.config.inbound.iter().any(|route| { + route.capability + == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + }) + }) + .ok_or_else(|| { + miette::miette!( + "mesh provision plan is missing the lowered framework.component controller target" + ) + }) +} + +fn local_site_controller_inbound_target( + artifact_root: &Path, + plan: &MeshProvisionPlan, + embedded_kind: Option, +) -> Result { + let target = site_controller_component_target(plan)?; + let peer_id = target.config.identity.id.clone(); + let mesh_port = target.config.mesh_listen.port(); + let peer_addr = match embedded_kind { + Some(EmbeddedMeshPlanKind::Compose) => { + compose_component_mesh_peer_addr(artifact_root, &peer_id, &target.output, mesh_port)? + } + Some(EmbeddedMeshPlanKind::Kubernetes) => { + kubernetes_component_mesh_peer_addr(artifact_root, &peer_id, &target.output, mesh_port)? + } + None => local_component_mesh_peer_addr(mesh_port), + }; + Ok(InboundTarget::MeshForward { + peer_addr, + peer_id: peer_id.clone(), + route_id: component_route_id( + &peer_id, + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + MeshProtocol::Http, + ), + capability: amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME.to_string(), + }) +} + fn build_dynamic_compose_route_overlay_payload( artifact_root: &Path, assigned_components: &[String], @@ -1519,8 +1565,11 @@ pub fn inject_site_controller_peer_router_routes( )); }; + let local_inbound_target = + local_site_controller_inbound_target(artifact_root, &plan, write_embedded)?; inject_site_controller_peer_router_routes_into_plan( &mut plan, + local_inbound_target, local_site_id, allowed_issuers, routes, @@ -1597,6 +1646,7 @@ fn set_compose_router_published_port( .wrap_err_with(|| format!("failed to write {}", path.display())) } +#[derive(Clone, Copy, Debug)] enum EmbeddedMeshPlanKind { Compose, Kubernetes, @@ -1604,6 +1654,7 @@ enum EmbeddedMeshPlanKind { fn inject_site_controller_peer_router_routes_into_plan( plan: &mut MeshProvisionPlan, + local_inbound_target: InboundTarget, local_site_id: &str, allowed_issuers: &[String], routes: &[SiteControllerPeerRouterRoute], @@ -1625,10 +1676,7 @@ fn inject_site_controller_peer_router_routes_into_plan( capability_profile: None, protocol: MeshProtocol::Http, http_plugins: Vec::new(), - target: InboundTarget::External { - url_env: amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV.to_string(), - optional: false, - }, + target: local_inbound_target, allowed_issuers: allowed_issuers.to_vec(), }); router diff --git a/runtime/site-controller/src/default_runtime/site_runtime_support.rs b/runtime/site-controller/src/default_runtime/site_runtime_support.rs index df3d6d8a..bb2118b6 100644 --- a/runtime/site-controller/src/default_runtime/site_runtime_support.rs +++ b/runtime/site-controller/src/default_runtime/site_runtime_support.rs @@ -74,7 +74,6 @@ pub(super) fn site_supervisor_plan_for_site_runtime( port_forward_mesh_port: None, port_forward_control_port: None, observability_endpoint: plan.observability_endpoint.clone(), - site_controller_plan_path: None, site_controller_url: None, launch_env: plan.launch_env.clone(), } @@ -1225,7 +1224,6 @@ mod tests { port_forward_mesh_port: None, port_forward_control_port: None, observability_endpoint: None, - site_controller_plan_path: None, site_controller_url: None, launch_env: BTreeMap::new(), } @@ -1420,7 +1418,6 @@ printf '%s' "${AMBER_VM_FORCE_TCG:-missing}" > "$1" kubernetes_consumer_router_mesh_addr: None, router_identity_id: Some(router_identity.id.clone()), router_public_key_b64: None, - site_controller_pid: None, site_controller_url: None, }, router_control: ControlEndpoint::Tcp("127.0.0.1:24100".to_string()), @@ -1503,7 +1500,6 @@ printf '%s' "${AMBER_VM_FORCE_TCG:-missing}" > "$1" router_public_key_b64: Some( base64::engine::general_purpose::STANDARD.encode(router_identity.public_key), ), - site_controller_pid: None, site_controller_url: None, }, router_control: ControlEndpoint::Tcp("127.0.0.1:24100".to_string()), @@ -1572,7 +1568,6 @@ printf '%s' "${AMBER_VM_FORCE_TCG:-missing}" > "$1" router_public_key_b64: Some( base64::engine::general_purpose::STANDARD.encode(router_identity.public_key), ), - site_controller_pid: None, site_controller_url: None, }, router_control: ControlEndpoint::Tcp("127.0.0.1:24100".to_string()), diff --git a/runtime/site-controller/src/http.rs b/runtime/site-controller/src/http.rs index 75455bed..38795791 100644 --- a/runtime/site-controller/src/http.rs +++ b/runtime/site-controller/src/http.rs @@ -9,21 +9,21 @@ use miette::{IntoDiagnostic as _, Result, WrapErr as _}; use serde::{Deserialize, Serialize}; use super::{ - orchestration::ProtocolApiError, planner::ControlStateApp, state::FRAMEWORK_AUTH_HEADER, *, + orchestration::ProtocolApiError, planner::ControlStateApp, state::CONTROL_STATE_AUTH_HEADER, *, }; pub(super) async fn cleanup_dynamic_bridge_proxies(app: &ControlStateApp) -> Result<()> { app.runtime.cleanup().await } -pub(super) fn authorize_framework_auth_header( +pub(super) fn authorize_control_state_auth_header( headers: &HeaderMap, expected: &str, ) -> std::result::Result<(), ProtocolApiError> { - let actual = required_header(headers, FRAMEWORK_AUTH_HEADER)?; + let actual = required_header(headers, CONTROL_STATE_AUTH_HEADER)?; if actual != expected { return Err(ProtocolApiError::unauthorized( - "invalid authenticated framework request header", + "invalid authenticated control-state request header", )); } Ok(()) @@ -41,7 +41,7 @@ pub(super) fn required_header( .map(ToOwned::to_owned) .ok_or_else(|| { ProtocolApiError::unauthorized(format!( - "missing authenticated framework request header `{name}`" + "missing authenticated control-state request header `{name}`" )) }) } diff --git a/runtime/site-controller/src/lib.rs b/runtime/site-controller/src/lib.rs index da69fb41..3ef9289f 100644 --- a/runtime/site-controller/src/lib.rs +++ b/runtime/site-controller/src/lib.rs @@ -97,9 +97,9 @@ pub use self::{ }, state::{ DynamicInputDirectRecord, DynamicInputRouteRecord, DynamicInputRouteTarget, - DynamicProxyExportRecord, SITE_CONTROLLER_INTERNAL_CAPABILITY, SITE_CONTROLLER_PORT, + DynamicProxyExportRecord, SITE_CONTROLLER_INTERNAL_CAPABILITY, SITE_CONTROLLER_SERVICE_NAME, SiteControllerPlan, authority_url_for_listen_addr, - build_site_controller_state, generate_framework_auth_token, + build_site_controller_state, generate_control_state_auth_token, site_controller_internal_route_id, write_control_state, write_site_controller_plan, }, }; diff --git a/runtime/site-controller/src/orchestration.rs b/runtime/site-controller/src/orchestration.rs index 2e8dd4ba..eddd44c4 100644 --- a/runtime/site-controller/src/orchestration.rs +++ b/runtime/site-controller/src/orchestration.rs @@ -186,7 +186,6 @@ pub(super) fn site_receipt_from_manager_state(state: &SiteManagerStateView) -> S kubernetes_consumer_router_mesh_addr: state.kubernetes_consumer_router_mesh_addr.clone(), router_identity_id: state.router_identity_id.clone(), router_public_key_b64: state.router_public_key_b64.clone(), - site_controller_pid: state.site_controller_pid, site_controller_url: state.site_controller_url.clone(), } } @@ -291,7 +290,6 @@ fn local_site_manager_state_view(app: &ControlStateApp) -> SiteManagerStateView .as_ref() .and_then(|state| state.router_public_key_b64.clone()) }), - site_controller_pid: None, site_controller_url: Some(app.controller_plan.authority_url.clone()), } } @@ -316,34 +314,6 @@ pub(super) fn load_launched_site( }) } -fn framework_route_overlay_id(authority_site_id: &str) -> String { - format!("framework-component-routes:{authority_site_id}") -} - -fn framework_component_route( - record: &CapabilityInstanceRecord, - allowed_issuers: Vec, - target: InboundTarget, -) -> InboundRoute { - InboundRoute { - route_id: record.route_id.clone(), - capability: record.capability.clone(), - capability_kind: Some("component".to_string()), - capability_profile: None, - protocol: MeshProtocol::Http, - http_plugins: Vec::new(), - target, - allowed_issuers, - } -} - -fn framework_controller_external_target() -> InboundTarget { - InboundTarget::External { - url_env: amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV.to_string(), - optional: false, - } -} - async fn peer_router_identity_for_overlay( app: &ControlStateApp, site_id: &str, @@ -356,7 +326,6 @@ async fn peer_router_identity_for_overlay( } let site_app = SiteControllerApp { control: app.clone(), - router_auth_token: app.control_state_auth_token.clone(), ready: Arc::new(std::sync::atomic::AtomicBool::new(true)), }; let deadline = tokio::time::Instant::now() + Duration::from_secs(30); @@ -378,25 +347,6 @@ async fn peer_router_identity_for_overlay( }) } -fn peer_router_mesh_addr_for_overlay( - app: &ControlStateApp, - site_id: &str, -) -> std::result::Result { - app.controller_plan - .peer_router_mesh_addrs - .get(site_id) - .cloned() - .ok_or_else(|| { - protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &format!( - "site controller `{}` has no mesh address for peer site `{site_id}`", - app.controller_plan.site_id - ), - ) - }) -} - fn local_router_control_endpoint( app: &ControlStateApp, ) -> std::result::Result { @@ -687,114 +637,6 @@ pub(super) async fn revoke_dynamic_capability_origin_overlays_local( Ok(()) } -pub(super) async fn framework_route_overlay_payload( - app: &ControlStateApp, -) -> std::result::Result, ProtocolErrorResponse> { - let state = app.control_state.lock().await.clone(); - let local_site_id = app.controller_plan.site_id.clone(); - let mut overlay = DesiredRouteOverlay { - peers: Vec::new(), - inbound_routes: Vec::new(), - }; - - for record in state.capability_instances.values() { - let authority_site_id = framework_authority_site_id(&state, record)?; - let consumer_site_id = record.recipient_site_id.clone(); - - if authority_site_id == local_site_id { - if consumer_site_id == local_site_id { - overlay.inbound_routes.push(framework_component_route( - record, - vec![record.recipient_peer_id.clone()], - framework_controller_external_target(), - )); - continue; - } - let consumer_peer = peer_router_identity_for_overlay(app, &consumer_site_id).await?; - if !overlay.peers.iter().any(|peer| peer.id == consumer_peer.id) { - overlay.peers.push(consumer_peer.clone()); - } - overlay.inbound_routes.push(framework_component_route( - record, - vec![consumer_peer.id], - framework_controller_external_target(), - )); - continue; - } - - if consumer_site_id != local_site_id { - continue; - } - - let authority_peer = peer_router_identity_for_overlay(app, &authority_site_id).await?; - let route = framework_component_route( - record, - vec![record.recipient_peer_id.clone()], - InboundTarget::MeshForward { - peer_addr: peer_router_mesh_addr_for_overlay(app, &authority_site_id)?, - peer_id: authority_peer.id.clone(), - route_id: record.route_id.clone(), - capability: record.capability.clone(), - }, - ); - if !overlay - .peers - .iter() - .any(|peer| peer.id == authority_peer.id) - { - overlay.peers.push(authority_peer.clone()); - } - overlay.inbound_routes.push(route); - } - - if overlay.peers.is_empty() && overlay.inbound_routes.is_empty() { - return Ok(None); - } - overlay.peers.sort_by(|left, right| left.id.cmp(&right.id)); - overlay - .inbound_routes - .sort_by(|left, right| left.route_id.cmp(&right.route_id)); - Ok(Some(overlay)) -} - -pub(super) async fn reconcile_local_framework_routes( - app: &ControlStateApp, -) -> std::result::Result<(), ProtocolErrorResponse> { - let overlay_id = framework_route_overlay_id(&app.controller_plan.site_id); - let endpoint = local_router_control_endpoint(app)?; - if let Some(overlay) = framework_route_overlay_payload(app).await? { - apply_route_overlay_with_retry( - &endpoint, - &overlay_id, - &overlay.peers, - &overlay.inbound_routes, - Duration::from_secs(30), - ) - .await - .map_err(|err| { - controller_protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &app.controller_plan.site_id, - "publish framework route overlay", - err, - ) - })?; - } else { - revoke_route_overlay_with_retry(&endpoint, &overlay_id, Duration::from_secs(30)) - .await - .map_err(|err| { - controller_protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &app.controller_plan.site_id, - "retract framework route overlay", - err, - ) - })?; - } - - Ok(()) -} - fn peer_site_router_url( app: &ControlStateApp, site_id: &str, @@ -2181,12 +2023,11 @@ pub(super) async fn execute_create_child( (tx_id, child) }; let (tx_id, child) = child; - reconcile_local_framework_routes(app).await?; let state = app.control_state.lock().await.clone(); if let Err(err) = prepare_child_on_sites(app, &state, &child).await { let rollback_err = rollback_prepared_sites(app, child.child_id).await; - let should_reconcile = { + { let mut state = app.control_state.lock().await; if state .pending_creates @@ -2204,13 +2045,7 @@ pub(super) async fn execute_create_child( Ok(()) }, )?; - true - } else { - false } - }; - if should_reconcile { - reconcile_local_framework_routes(app).await?; } let err = if let Err(rollback_err) = rollback_err { protocol_error( @@ -2265,7 +2100,7 @@ pub(super) async fn execute_destroy_child( child_name: &str, ) -> std::result::Result<(), ProtocolApiError> { let _authority_guard = acquire_authority_lock(app, authority_realm_id).await; - let (next, reconcile_routes) = { + let next = { let mut state = app.control_state.lock().await; let Some(child) = state .live_children @@ -2291,10 +2126,10 @@ pub(super) async fn execute_destroy_child( Ok(()) }, )?; - ((child.child_id, ChildState::DestroyRequested), true) + (child.child_id, ChildState::DestroyRequested) } - ChildState::DestroyRequested => ((child.child_id, ChildState::DestroyRequested), false), - ChildState::DestroyRetracted => ((child.child_id, ChildState::DestroyRetracted), false), + ChildState::DestroyRequested => (child.child_id, ChildState::DestroyRequested), + ChildState::DestroyRetracted => (child.child_id, ChildState::DestroyRetracted), ChildState::DestroyCommitted | ChildState::CreateAborted => return Ok(()), _ => { return Err(protocol_error( @@ -2305,9 +2140,6 @@ pub(super) async fn execute_destroy_child( } } }; - if reconcile_routes { - reconcile_local_framework_routes(app).await?; - } match next.1 { ChildState::DestroyRequested => continue_destroy_requested(app, next.0).await?, ChildState::DestroyRetracted => continue_destroy_retracted(app, next.0).await?, @@ -2389,9 +2221,6 @@ pub(super) async fn recover_control_state(app: &ControlStateApp) -> Result<()> { ChildState::CreateAborted | ChildState::DestroyCommitted => {} } } - reconcile_local_framework_routes(app) - .await - .map_err(|err| miette::miette!(err.message))?; Ok(()) } diff --git a/runtime/site-controller/src/planner.rs b/runtime/site-controller/src/planner.rs index b8cd886f..cc038b2e 100644 --- a/runtime/site-controller/src/planner.rs +++ b/runtime/site-controller/src/planner.rs @@ -69,6 +69,19 @@ pub(super) async fn prepare_child_record( BTreeMap::from([(wrapper_url.to_string(), wrapper_manifest)]), ) .await?; + let compiled = amber_compiler::run_plan::lower_framework_component_bindings_for_placement( + &compiled, + &placement_file_from_state(state), + ) + .map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "failed to lower framework.component bindings in dynamic child `{}`: {err}", + request.name + ), + ) + })?; let (fragment, outputs) = extract_live_child_fragment( state, &compiled, @@ -1034,24 +1047,12 @@ pub(super) fn build_wrapper_manifest( synthetic_sources.insert( synthetic_name.clone(), SyntheticSourceRecord { - slot_name: binding.slot_name.clone(), actual_source, source_child_id, weak: source.weak, }, ); - if let BindingFrom::Framework(framework) = &source.from { - bindings.push(raw_binding( - &format!("#{child_name}"), - binding.slot_name.clone(), - "framework", - framework.capability.to_string(), - source.weak, - )?); - continue; - } - slots.insert( synthetic_name.clone(), SlotDecl::builder() @@ -1988,25 +1989,6 @@ pub(super) fn extract_live_child_fragment( rewritten.weak = synthetic.weak; synthetic.source_child_id } - BindingFromIr::Framework { - authority_realm, .. - } if *authority_realm == wrapper_root => { - let Some(synthetic) = synthetic_sources.values().find(|source| { - source.slot_name == binding.to.slot - && matches!(source.actual_source, BindingFrom::Framework(_)) - }) else { - return Err(protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &format!( - "missing framework synthetic source mapping for slot `{}`", - binding.to.slot - ), - )); - }; - rewritten.from = BindingFromIr::from(&synthetic.actual_source); - rewritten.weak = synthetic.weak; - synthetic.source_child_id - } BindingFromIr::Framework { authority_realm, .. } => { @@ -2154,6 +2136,17 @@ pub(super) fn collect_capability_instances( .iter() .map(|component| (component.id, component.moniker.clone())) .collect::>(); + let mut controller_metadata_by_id = state + .base_scenario + .components + .iter() + .filter_map(|component| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .map(|metadata| (component.id, metadata)) + }) + .collect::>(); for child in &active_children { let Some(fragment) = child.fragment.as_ref() else { continue; @@ -2164,6 +2157,12 @@ pub(super) fn collect_capability_instances( .iter() .map(|component| (component.id, component.moniker.clone())), ); + controller_metadata_by_id.extend(fragment.components.iter().filter_map(|component| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .map(|metadata| (component.id, metadata)) + })); } let mut site_by_moniker = state.placement.assignments.clone(); @@ -2178,6 +2177,7 @@ pub(super) fn collect_capability_instances( binding, &moniker_by_id, &site_by_moniker, + &controller_metadata_by_id, state.generation, )?; } @@ -2191,6 +2191,7 @@ pub(super) fn collect_capability_instances( &binding.binding, &moniker_by_id, &site_by_moniker, + &controller_metadata_by_id, state.generation, )?; } @@ -2203,68 +2204,90 @@ pub(super) fn collect_capability_instance_from_binding( binding: &BindingIr, moniker_by_id: &BTreeMap, site_by_moniker: &BTreeMap, + controller_metadata_by_id: &BTreeMap< + usize, + amber_compiler::run_plan::FrameworkComponentControllerMetadata, + >, generation: u64, ) -> Result<()> { - let BindingFromIr::Framework { - authority_realm, - capability, - } = &binding.from - else { - return Ok(()); - }; - if capability != "component" { - return Ok(()); - } - - let authority_realm_moniker = moniker_by_id.get(authority_realm).cloned().ok_or_else(|| { - miette::miette!( - "framework.component authority realm id {authority_realm} is missing from the \ - authoritative live graph" - ) - })?; let recipient_component_moniker = moniker_by_id .get(&binding.to.component) .cloned() .ok_or_else(|| { miette::miette!( - "framework.component recipient component id {} is missing from the authoritative \ - live graph", - binding.to.component - ) + "framework.component recipient component id {} is missing from the \ + authoritative live graph", + binding.to.component + ) })?; let recipient_site_id = site_by_moniker .get(&recipient_component_moniker) .cloned() .ok_or_else(|| { miette::miette!( - "framework.component recipient `{recipient_component_moniker}` is missing a site \ - assignment in the authoritative live graph" - ) + "framework.component recipient `{recipient_component_moniker}` is missing a \ + site assignment in the authoritative live graph" + ) })?; - let cap_instance_id = framework_cap_instance_id( - authority_realm_moniker.as_str(), - recipient_component_moniker.as_str(), - &binding.to.component.to_string(), - &binding.to.slot, - capability, - ); - records.insert( - cap_instance_id.clone(), - CapabilityInstanceRecord { - cap_instance_id: cap_instance_id.clone(), - route_id: cap_instance_id, - authority_realm_id: *authority_realm, - authority_realm_moniker, - recipient_component_id: binding.to.component, - recipient_component_moniker: recipient_component_moniker.clone(), - recipient_peer_id: recipient_component_moniker, - recipient_site_id, - capability: capability.clone(), - slot: binding.to.slot.clone(), - generation, - }, - ); - Ok(()) + + match &binding.from { + BindingFromIr::Framework { capability, .. } => { + if capability != "component" { + return Ok(()); + } + Err(miette::miette!( + "framework.component binding {}.{} must be lowered before capability refresh", + recipient_component_moniker, + binding.to.slot, + )) + } + BindingFromIr::Component { component, provide } => { + let Some(metadata) = controller_metadata_by_id.get(component) else { + return Ok(()); + }; + let Some(grant) = metadata.grants.get(provide) else { + return Ok(()); + }; + let provider_moniker = moniker_by_id.get(component).cloned().ok_or_else(|| { + miette::miette!( + "framework.component controller component id {component} is missing from \ + the authoritative live graph" + ) + })?; + let route_id = amber_mesh::component_route_id( + provider_moniker.as_str(), + provide, + amber_mesh::MeshProtocol::Http, + ); + let cap_instance_id = framework_cap_instance_id( + grant.authority_realm_moniker.as_str(), + recipient_component_moniker.as_str(), + &binding.to.component.to_string(), + &binding.to.slot, + "component", + ); + records.insert( + cap_instance_id.clone(), + CapabilityInstanceRecord { + cap_instance_id, + route_id, + authority_realm_id: grant.authority_realm_id, + authority_realm_moniker: grant.authority_realm_moniker.clone(), + recipient_component_id: binding.to.component, + recipient_component_moniker: recipient_component_moniker.clone(), + recipient_peer_id: recipient_component_moniker, + recipient_site_id, + controller_site_id: metadata.execution_site.clone(), + managed_site_id: grant.managed_site.clone(), + capability: "component".to_string(), + slot: binding.to.slot.clone(), + generation, + }, + ); + Ok(()) + } + _ => Ok(()), + } } pub(super) fn template_mode(template: &ChildTemplate) -> TemplateMode { @@ -3276,7 +3299,6 @@ pub(super) struct ControlStateApp { #[derive(Clone)] pub(super) struct SiteControllerApp { pub(super) control: ControlStateApp, - pub(super) router_auth_token: Arc, pub(super) ready: Arc, } @@ -3315,7 +3337,5 @@ pub(super) struct SiteManagerStateView { #[serde(default)] pub(super) router_public_key_b64: Option, #[serde(default)] - pub(super) site_controller_pid: Option, - #[serde(default)] pub(super) site_controller_url: Option, } diff --git a/runtime/site-controller/src/runtime_api.rs b/runtime/site-controller/src/runtime_api.rs index 306dd132..a035e4d7 100644 --- a/runtime/site-controller/src/runtime_api.rs +++ b/runtime/site-controller/src/runtime_api.rs @@ -149,8 +149,6 @@ pub struct SiteReceipt { #[serde(default, skip_serializing_if = "Option::is_none")] pub router_public_key_b64: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub site_controller_pid: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] pub site_controller_url: Option, } diff --git a/runtime/site-controller/src/site_controller.rs b/runtime/site-controller/src/site_controller.rs index a2ee5054..727724d3 100644 --- a/runtime/site-controller/src/site_controller.rs +++ b/runtime/site-controller/src/site_controller.rs @@ -27,10 +27,7 @@ use super::{ ControlDynamicRevokeRequest, ControlDynamicShareRequest, InternalDynamicResolveOriginRequest, }, - http::{ - authorize_framework_auth_header, cleanup_dynamic_bridge_proxies, read_json, - required_header, shutdown_signal, - }, + http::{cleanup_dynamic_bridge_proxies, read_json, required_header, shutdown_signal}, orchestration::{ ClearExportPeerOverlayRequest, ClearExternalSlotOverlayRequest, ProtocolApiError, PublishExportPeerOverlayRequest, PublishExternalSlotOverlayRequest, @@ -69,7 +66,7 @@ pub(crate) async fn run_site_controller( run_root: PathBuf::from(&plan.run_root), state_root: PathBuf::from(&plan.state_root), mesh_scope: Arc::::from(plan.mesh_scope.clone()), - control_state_auth_token: Arc::::from(plan.auth_token.clone()), + control_state_auth_token: Arc::::from(plan.control_state_auth_token.clone()), controller_plan: Arc::new(plan.clone()), authority_locks: Arc::new(Mutex::new(BTreeMap::new())), runtime, @@ -77,7 +74,6 @@ pub(crate) async fn run_site_controller( let ready = Arc::new(std::sync::atomic::AtomicBool::new(false)); let app_state = SiteControllerApp { control: control.clone(), - router_auth_token: Arc::::from(plan.auth_token), ready: ready.clone(), }; let app = site_controller_router(app_state.clone()); @@ -238,7 +234,7 @@ async fn get_site_controller_state_route( State(app): State, headers: HeaderMap, ) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + authorize_local_controller_request(&app, &headers)?; Ok(Json(app.control.control_state.lock().await.clone())) } @@ -248,16 +244,35 @@ pub(crate) struct RouterIdentityResponse { pub(crate) public_key_b64: String, } -fn authorize_local_controller_request( +pub(super) fn authorize_local_controller_request( app: &SiteControllerApp, headers: &HeaderMap, ) -> std::result::Result<(), ProtocolApiError> { - authorize_framework_auth_header(headers, app.router_auth_token.as_ref())?; if !controller_local_only(headers) { return Err(ProtocolApiError::unauthorized( "local controller endpoint requires router-local forwarding".to_string(), )); } + let route_id = required_header(headers, FRAMEWORK_ROUTE_ID_HEADER)?; + let expected_route_id = site_controller_internal_route_id(&app.control.controller_plan.site_id); + if route_id != expected_route_id { + return Err(ProtocolApiError::unauthorized(format!( + "local controller endpoint requires internal route `{expected_route_id}`" + ))); + } + let peer_id = required_header(headers, FRAMEWORK_PEER_ID_HEADER)?; + let local_router_id = app.control.controller_plan.router_identity_id.as_str(); + let known_peer_router = app + .control + .controller_plan + .peer_router_identities + .values() + .any(|identity| identity.id == peer_id); + if peer_id != local_router_id && !known_peer_router { + return Err(ProtocolApiError::unauthorized(format!( + "local controller endpoint received an unknown router peer `{peer_id}`" + ))); + } Ok(()) } @@ -314,16 +329,43 @@ pub(super) async fn authorize_public_request( app: &SiteControllerApp, headers: &HeaderMap, ) -> std::result::Result<(CapabilityInstanceRecord, FrameworkControlState), ProtocolApiError> { - authorize_framework_auth_header(headers, app.router_auth_token.as_ref())?; let route_id = required_header(headers, FRAMEWORK_ROUTE_ID_HEADER)?; let peer_id = required_header(headers, FRAMEWORK_PEER_ID_HEADER)?; let state = app.control.control_state.lock().await.clone(); - let record = super::api::authorize_capability_instance(&state, &route_id, &peer_id) + let record = super::api::authorize_capability_route(&state, &route_id, &peer_id) .map_err(ProtocolApiError::from)? .clone(); Ok((record, state)) } +pub(super) fn authorize_dynamic_caps_sidecar_request( + _app: &SiteControllerApp, + headers: &HeaderMap, + expected_component_id: &str, +) -> std::result::Result<(), ProtocolApiError> { + let route_id = required_header(headers, FRAMEWORK_ROUTE_ID_HEADER)?; + if !route_id.ends_with(&format!( + ":{}:http", + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + )) { + return Err(ProtocolApiError::unauthorized( + "dynamic capability control requests must arrive over the site controller internal \ + route" + .to_string(), + )); + } + let peer_id = required_header(headers, FRAMEWORK_PEER_ID_HEADER)?; + let expected_peer_id = dynamic_caps::moniker_from_logical_component_id(expected_component_id) + .map_err(ProtocolApiError::from)?; + if peer_id != expected_peer_id { + return Err(ProtocolApiError::unauthorized(format!( + "dynamic capability control request for `{expected_component_id}` came from \ + `{peer_id}`" + ))); + } + Ok(()) +} + fn controller_local_only(headers: &HeaderMap) -> bool { headers .get(CONTROLLER_LOCAL_ONLY_HEADER) @@ -700,10 +742,19 @@ fn site_for_dynamic_ref( } fn framework_request_site_id( - state: &FrameworkControlState, + _state: &FrameworkControlState, record: &CapabilityInstanceRecord, ) -> std::result::Result { - framework_authority_site_id(state, record).map_err(ProtocolApiError::from) + if !record.controller_site_id.is_empty() { + return Ok(record.controller_site_id.clone()); + } + Err(ProtocolApiError::from(protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "framework.component capability instance `{}` is missing its controller_site_id", + record.cap_instance_id + ), + ))) } pub(super) async fn execute_site_controller_framework_inspect( @@ -716,8 +767,8 @@ pub(super) async fn execute_site_controller_framework_inspect( let authority_site = framework_request_site_id(state, record)?; if authority_site != app.control.controller_plan.site_id { return Err(ProtocolApiError::control_state_unavailable(format!( - "framework.component request for authority site `{authority_site}` reached site \ - controller `{}`; router framework route overlays are stale", + "framework.component request is bound to controller site `{authority_site}` but \ + reached site controller `{}`", app.control.controller_plan.site_id ))); } @@ -737,8 +788,8 @@ pub(super) async fn execute_site_controller_framework_mutate( let authority_site = framework_request_site_id(state, record)?; if authority_site != app.control.controller_plan.site_id { return Err(ProtocolApiError::control_state_unavailable(format!( - "framework.component request for authority site `{authority_site}` reached site \ - controller `{}`; router framework route overlays are stale", + "framework.component request is bound to controller site `{authority_site}` but \ + reached site controller `{}`", app.control.controller_plan.site_id ))); } @@ -1244,7 +1295,7 @@ async fn control_dynamic_held_list_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + authorize_dynamic_caps_sidecar_request(&app, &headers, &request.holder_component_id)?; match execute_site_controller_dynamic_caps_inspect( &app, DynamicCapsInspectRequest::HeldList(request), @@ -1262,7 +1313,7 @@ async fn control_dynamic_held_detail_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + authorize_dynamic_caps_sidecar_request(&app, &headers, &request.holder_component_id)?; match execute_site_controller_dynamic_caps_inspect( &app, DynamicCapsInspectRequest::HeldDetail(request), @@ -1280,7 +1331,7 @@ async fn control_dynamic_share_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + authorize_dynamic_caps_sidecar_request(&app, &headers, &request.caller_component_id)?; match execute_site_controller_dynamic_caps_mutate( &app, DynamicCapsMutateRequest::Share(request), @@ -1298,7 +1349,7 @@ async fn control_dynamic_grant_authorities_sync_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + authorize_local_controller_request(&app, &headers)?; Ok(Json( local_sync_dynamic_grant_authorities(&app, request).await?, )) @@ -1309,7 +1360,7 @@ async fn control_dynamic_inspect_ref_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + authorize_dynamic_caps_sidecar_request(&app, &headers, &request.holder_component_id)?; match execute_site_controller_dynamic_caps_inspect( &app, DynamicCapsInspectRequest::InspectRef(request), @@ -1327,7 +1378,7 @@ async fn control_dynamic_revoke_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + authorize_dynamic_caps_sidecar_request(&app, &headers, &request.caller_component_id)?; match execute_site_controller_dynamic_caps_mutate( &app, DynamicCapsMutateRequest::Revoke(request), @@ -1346,7 +1397,7 @@ async fn control_dynamic_resolve_origin_route( Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_framework_auth_header(&headers, app.control.control_state_auth_token.as_ref())?; + authorize_dynamic_caps_sidecar_request(&app, &headers, &request.holder_component_id)?; match execute_site_controller_dynamic_caps_inspect( &app, DynamicCapsInspectRequest::ResolveOrigin(request), diff --git a/runtime/site-controller/src/state.rs b/runtime/site-controller/src/state.rs index a232f97a..0083940e 100644 --- a/runtime/site-controller/src/state.rs +++ b/runtime/site-controller/src/state.rs @@ -8,10 +8,9 @@ pub(super) const SITE_CONTROLLER_PLAN_VERSION: u32 = 1; pub(super) const SITE_CONTROLLER_STATE_PATH: &str = "/v1/controller/state"; pub(super) const FRAMEWORK_ROUTE_ID_HEADER: &str = "x-amber-route-id"; pub(super) const FRAMEWORK_PEER_ID_HEADER: &str = "x-amber-peer-id"; -pub(super) const FRAMEWORK_AUTH_HEADER: &str = "x-amber-framework-auth"; +pub(super) const CONTROL_STATE_AUTH_HEADER: &str = "x-amber-control-state-auth"; pub const SITE_CONTROLLER_INTERNAL_CAPABILITY: &str = "amber.internal.site_controller"; pub const SITE_CONTROLLER_SERVICE_NAME: &str = "amber-site-controller"; -pub const SITE_CONTROLLER_PORT: u16 = 4100; pub fn site_controller_internal_route_id(site_id: &str) -> String { format!("site-controller:{site_id}") @@ -46,6 +45,10 @@ pub(crate) struct CapabilityInstanceRecord { pub(crate) recipient_component_moniker: String, pub(crate) recipient_peer_id: String, pub(crate) recipient_site_id: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub(crate) controller_site_id: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub(crate) managed_site_id: String, pub(crate) capability: String, pub(crate) slot: String, pub(crate) generation: u64, @@ -282,7 +285,7 @@ pub struct SiteControllerPlan { pub state_root: String, pub site_state_root: String, pub artifact_dir: String, - pub auth_token: String, + pub control_state_auth_token: String, pub dynamic_caps_token_verify_key_b64: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub storage_root: Option, @@ -1073,7 +1076,7 @@ fn remap_output_handle_for_snapshot( }) } -pub fn generate_framework_auth_token(mesh_scope: &str, purpose: &str) -> String { +pub fn generate_control_state_auth_token(mesh_scope: &str, purpose: &str) -> String { base64::engine::general_purpose::STANDARD.encode( MeshIdentity::generate( format!("/framework/{purpose}"), @@ -1249,7 +1252,7 @@ pub fn write_site_controller_plan( state_root: &Path, site_state_root: &Path, artifact_dir: &Path, - auth_token: &str, + control_state_auth_token: &str, dynamic_caps_token_verify_key_b64: &str, storage_root: Option<&str>, runtime_root: Option<&str>, @@ -1283,7 +1286,7 @@ pub fn write_site_controller_plan( state_root: state_root.display().to_string(), site_state_root: site_state_root.display().to_string(), artifact_dir: artifact_dir.display().to_string(), - auth_token: auth_token.to_string(), + control_state_auth_token: control_state_auth_token.to_string(), dynamic_caps_token_verify_key_b64: dynamic_caps_token_verify_key_b64.to_string(), storage_root: storage_root.map(str::to_string), runtime_root: runtime_root.map(str::to_string), @@ -1298,72 +1301,6 @@ pub fn write_site_controller_plan( Ok(plan) } -pub(crate) fn site_id_for_authority_realm( - state: &FrameworkControlState, - authority_realm_id: usize, -) -> std::result::Result { - let authority_moniker = decode_live_scenario(state)? - .component(ComponentId(authority_realm_id)) - .moniker - .to_string(); - site_id_for_moniker( - state, - authority_moniker.as_str(), - &format!("authority realm `{authority_moniker}`"), - ) -} - -pub(crate) fn framework_authority_site_id( - state: &FrameworkControlState, - record: &CapabilityInstanceRecord, -) -> std::result::Result { - if let Ok(site_id) = site_id_for_authority_realm(state, record.authority_realm_id) { - return Ok(site_id); - } - let authority = record.authority_realm_moniker.as_str(); - let recipient = record.recipient_component_moniker.as_str(); - if authority != "/" - && recipient != authority - && !recipient.starts_with(&format!("{authority}/")) - { - return Err(protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &format!( - "framework.component recipient `{recipient}` is not inside authority realm \ - `{authority}`" - ), - )); - } - let assignments = planned_component_site_map(state); - let recipient_segments = recipient - .trim_start_matches('/') - .split('/') - .filter(|segment| !segment.is_empty()) - .collect::>(); - let authority_depth = authority - .trim_matches('/') - .split('/') - .filter(|segment| !segment.is_empty()) - .count(); - for depth in authority_depth..=recipient_segments.len() { - let moniker = if depth == 0 { - "/".to_string() - } else { - format!("/{}", recipient_segments[..depth].join("/")) - }; - if let Some(site_id) = assignments.get(&moniker) { - return Ok(site_id.clone()); - } - } - Err(protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &format!( - "framework.component authority realm `{authority}` has no assigned component on the \ - path to recipient `{recipient}`" - ), - )) -} - pub(crate) fn site_id_for_dynamic_grant( state: &FrameworkControlState, grant_id: &str, diff --git a/runtime/site-controller/src/tests.rs b/runtime/site-controller/src/tests.rs index 18547e24..978cd76d 100644 --- a/runtime/site-controller/src/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -1227,6 +1227,7 @@ async fn compile_dynamic_caps_binding_state() -> FrameworkControlState { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -1669,6 +1670,7 @@ async fn compile_dynamic_caps_external_root_state() -> FrameworkControlState { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -1768,6 +1770,7 @@ async fn same_site_dynamic_child_output_bindings_reuse_provider_component_routes SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -1926,6 +1929,7 @@ async fn same_site_static_child_export_bindings_reuse_provider_component_routes( SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -2011,6 +2015,7 @@ async fn dynamic_grant_routes_to_holder_site_not_offered_site_order() { SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, ); let share = super::dynamic_caps::share_dynamic_capability( @@ -2057,6 +2062,7 @@ async fn dynamic_caps_cross_site_share_syncs_holder_authority_through_site_route SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ( @@ -2064,6 +2070,7 @@ async fn dynamic_caps_cross_site_share_syncs_holder_authority_through_site_route SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ]), @@ -2140,7 +2147,6 @@ async fn dynamic_caps_cross_site_share_syncs_holder_authority_through_site_route BTreeMap::from([("direct_b".to_string(), holder_base_url)]); let controller_app = SiteControllerApp { control: app, - router_auth_token: Arc::::from("test-router-auth"), ready: ready_site_controller_flag(), }; @@ -2200,6 +2206,7 @@ async fn compile_empty_control_state() -> (TempDir, FrameworkControlState, PathB SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -2216,6 +2223,80 @@ async fn compile_empty_control_state() -> (TempDir, FrameworkControlState, PathB (dir, state, state_path) } +#[tokio::test] +async fn wrapper_manifest_lowers_framework_sources_to_synthetic_slots() { + let (dir, mut state, _) = compile_empty_control_state().await; + let child_path = dir.path().join("child.json5"); + state.base_scenario.manifest_catalog.insert( + "child".to_string(), + amber_scenario::ir::ManifestCatalogEntryIr { + source_ref: file_url(&child_path), + digest: amber_manifest::ManifestDigest::new([7; 32]), + manifest: amber_manifest::Manifest::empty(), + }, + ); + + let (manifest, synthetic_sources) = build_wrapper_manifest( + &state, + &ResolvedTemplateContract { + config: BTreeMap::new(), + bindings: BTreeMap::new(), + slot_decls: BTreeMap::new(), + visible_exports: Vec::new(), + }, + "child", + "child", + None, + &[ResolvedTemplateBinding { + slot_name: "creator".to_string(), + slot_decl: amber_manifest::SlotDecl::builder() + .decl( + amber_manifest::CapabilityDecl::builder() + .kind(amber_manifest::CapabilityKind::Http) + .build(), + ) + .build(), + sources: vec![ResolvedBindingSource { + from: BindingFrom::Framework(amber_scenario::FrameworkRef { + authority: amber_scenario::ComponentId(state.base_scenario.root), + capability: amber_manifest::FrameworkCapabilityName::try_from("component") + .expect("framework capability name should parse"), + }), + weak: true, + }], + source_child_id: None, + }], + ) + .expect("wrapper manifest should build"); + + assert!( + manifest.bindings().iter().all(|binding| !matches!( + binding.binding.from, + amber_manifest::BindingSource::Framework(_) + )), + "wrapper manifest should not recreate raw framework bindings: {:?}", + manifest.bindings() + ); + assert_eq!( + manifest.slots().len(), + 1, + "framework sources should lower to synthetic wrapper slots" + ); + assert_eq!( + synthetic_sources.len(), + 1, + "framework source mapping should still be recorded" + ); + assert!(matches!( + synthetic_sources + .values() + .next() + .expect("synthetic source should exist") + .actual_source, + BindingFrom::Framework(_) + )); +} + async fn compile_exact_template_control_state() -> (TempDir, FrameworkControlState, PathBuf) { let dir = TempDir::new().expect("temp dir"); let root_path = dir.path().join("root.json5"); @@ -2261,6 +2342,7 @@ async fn compile_exact_template_control_state() -> (TempDir, FrameworkControlSta SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -2347,6 +2429,7 @@ async fn compile_framework_binding_control_state() -> ( SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -2496,6 +2579,7 @@ fn test_control_state_app( SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, )]) } else { @@ -2567,7 +2651,7 @@ fn test_control_state_app( state_root: state_root.display().to_string(), site_state_root: site_state_root.display().to_string(), artifact_dir: artifact_dir.display().to_string(), - auth_token: "test-control-state-auth".to_string(), + control_state_auth_token: "test-control-state-auth".to_string(), dynamic_caps_token_verify_key_b64: String::new(), storage_root: Some(storage_root.display().to_string()), runtime_root: Some(runtime_root.display().to_string()), @@ -2810,7 +2894,6 @@ async fn install_framework_site_controller_fixture( "router_mesh_addr": format!("127.0.0.1:{}", plan.router_mesh_port.expect("router mesh port")), "router_identity_id": plan.router_identity_id, "router_public_key_b64": base64::engine::general_purpose::STANDARD.encode([11; 32]), - "site_controller_pid": 1, "site_controller_url": plan.authority_url, }), ) @@ -3056,7 +3139,6 @@ struct FrameworkMcpHarness { base_url: String, route_id: String, peer_id: String, - auth_token: String, handles: Vec>, } @@ -3079,10 +3161,8 @@ impl FrameworkMcpHarness { } else { Vec::new() }; - let auth_token = "test-router-auth".to_string(); let controller_app = SiteControllerApp { control: app, - router_auth_token: Arc::::from(auth_token.clone()), ready: ready_site_controller_flag(), }; handles.push(tokio::spawn(async move { @@ -3098,16 +3178,14 @@ impl FrameworkMcpHarness { _dir: dir, client: Client::new(), base_url, - route_id: record.cap_instance_id, + route_id: record.route_id, peer_id: record.recipient_peer_id, - auth_token, handles, } } fn http_headers(&self) -> Vec<(String, String)> { vec![ - (FRAMEWORK_AUTH_HEADER.to_string(), self.auth_token.clone()), (FRAMEWORK_ROUTE_ID_HEADER.to_string(), self.route_id.clone()), (FRAMEWORK_PEER_ID_HEADER.to_string(), self.peer_id.clone()), ] @@ -3371,7 +3449,6 @@ async fn framework_component_rejects_stale_nonlocal_controller_delivery() { authority_locks: app.authority_locks.clone(), runtime: app.runtime.clone(), }, - router_auth_token: Arc::::from("test-router-auth"), ready: ready_site_controller_flag(), }; @@ -3388,9 +3465,7 @@ async fn framework_component_rejects_stale_nonlocal_controller_delivery() { }; assert_eq!(err.0.code, ProtocolErrorCode::ControlStateUnavailable); assert!( - err.0 - .message - .contains("router framework route overlays are stale"), + err.0.message.contains("bound to controller site"), "unexpected error: {}", err.0.message ); @@ -3401,7 +3476,6 @@ async fn framework_component_rejects_requests_while_controller_recovers() { let (_dir, state, state_path, record) = compile_framework_binding_control_state().await; let controller_app = SiteControllerApp { control: test_control_state_app(&_dir, state.clone(), state_path), - router_auth_token: Arc::::from("test-router-auth"), ready: Arc::new(std::sync::atomic::AtomicBool::new(false)), }; @@ -3422,210 +3496,27 @@ async fn framework_component_rejects_requests_while_controller_recovers() { } #[tokio::test] -async fn framework_component_cross_site_routes_forward_through_site_routers() { - let (dir, mut state, _state_path, _) = compile_framework_binding_control_state().await; - state.placement.offered_sites = BTreeMap::from([ - ( - "authority".to_string(), - SiteDefinition { - kind: SiteKind::Direct, - context: None, - }, - ), - ( - "consumer".to_string(), - SiteDefinition { - kind: SiteKind::Direct, - context: None, - }, - ), - ]); - state.placement.defaults = PlacementDefaults { - path: Some("authority".to_string()), - ..PlacementDefaults::default() - }; - state.placement.placement_components = BTreeMap::from([ - ("/".to_string(), "authority".to_string()), - ("/admin".to_string(), "consumer".to_string()), - ]); - state.placement.assignments = state.placement.placement_components.clone(); - refresh_capability_instances(&mut state).expect("framework routes should refresh"); - let state_path = dir.path().join("control-state.json"); - write_control_state(&state_path, &state).expect("state should write"); - let app = test_control_state_app(&dir, state, state_path); - let router_public_key_b64 = base64::engine::general_purpose::STANDARD.encode([9u8; 32]); - let authority_router = Router::new().route( - "/v1/controller/router-identity", - axum::routing::get({ - let router_public_key_b64 = router_public_key_b64.clone(); - move || { - let router_public_key_b64 = router_public_key_b64.clone(); - async move { - Json(RouterIdentityResponse { - id: "/site/authority/router".to_string(), - public_key_b64: router_public_key_b64, - }) - } - } - }), - ); - let consumer_router = Router::new().route( - "/v1/controller/router-identity", - axum::routing::get({ - let router_public_key_b64 = router_public_key_b64.clone(); - move || { - let router_public_key_b64 = router_public_key_b64.clone(); - async move { - Json(RouterIdentityResponse { - id: "/site/consumer/router".to_string(), - public_key_b64: router_public_key_b64, - }) - } - } - }), - ); - let (authority_base_url, _authority_handle) = spawn_test_router(authority_router).await; - let (consumer_base_url, _consumer_handle) = spawn_test_router(consumer_router).await; - - let mut authority_plan = app.controller_plan.as_ref().clone(); - authority_plan.peer_site_router_urls = - BTreeMap::from([("consumer".to_string(), consumer_base_url)]); - let authority_app = ControlStateApp { - controller_plan: Arc::new(authority_plan), - ..app.clone() - }; - - let mut consumer_plan = app.controller_plan.as_ref().clone(); - consumer_plan.site_id = "consumer".to_string(); - consumer_plan.router_identity_id = "/site/consumer/router".to_string(); - consumer_plan.peer_site_router_urls = - BTreeMap::from([("authority".to_string(), authority_base_url)]); - consumer_plan.peer_router_mesh_addrs = - BTreeMap::from([("authority".to_string(), "127.0.0.1:24000".to_string())]); - let consumer_app = ControlStateApp { - controller_plan: Arc::new(consumer_plan), - ..app.clone() - }; - - let authority_overlay = framework_route_overlay_payload(&authority_app) - .await - .expect("authority framework routes should materialize") - .expect("authority site should get a framework route overlay"); - let consumer_overlay = framework_route_overlay_payload(&consumer_app) - .await - .expect("consumer framework routes should materialize") - .expect("consumer site should get a framework route overlay"); - - assert!( - authority_overlay - .peers - .iter() - .any(|peer| peer.id == "/site/consumer/router"), - "authority router should accept framework traffic from the consumer router", - ); - assert!( - consumer_overlay - .peers - .iter() - .any(|peer| peer.id == "/site/authority/router"), - "consumer router should forward framework traffic to the authority router", - ); - - assert!( - consumer_overlay.inbound_routes.iter().any(|route| matches!( - &route.target, - InboundTarget::MeshForward { - peer_id, - peer_addr, - route_id, - capability, - } if peer_id == "/site/authority/router" - && peer_addr == "127.0.0.1:24000" - && route_id == &route.route_id - && capability == &route.capability - )), - "cross-site framework requests must enter the consumer router and cross the router mesh", - ); - - assert!( - authority_overlay - .inbound_routes - .iter() - .any(|route| matches!( - &route.target, - InboundTarget::External { url_env, optional } - if url_env == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_URL_ENV && !optional - )), - "the authority router should hand framework requests to its local site controller only \ - after the router hop", - ); -} - -#[tokio::test] -async fn framework_route_overlay_payload_uses_planned_peer_router_identities() { - let (dir, mut state, state_path, _) = compile_framework_binding_control_state().await; - state.placement.offered_sites = BTreeMap::from([ - ( - "authority".to_string(), - SiteDefinition { - kind: SiteKind::Direct, - context: None, - }, - ), - ( - "consumer".to_string(), - SiteDefinition { - kind: SiteKind::Direct, - context: None, - }, - ), - ]); - state.placement.defaults = PlacementDefaults { - path: Some("authority".to_string()), - ..PlacementDefaults::default() - }; - state.placement.placement_components = BTreeMap::from([ - ("/".to_string(), "authority".to_string()), - ("/admin".to_string(), "consumer".to_string()), - ]); - state.placement.assignments = state.placement.placement_components.clone(); - refresh_capability_instances(&mut state).expect("framework routes should refresh"); - write_control_state(&state_path, &state).expect("state should write"); - let app = test_control_state_app(&dir, state, state_path); - - let mut consumer_plan = app.controller_plan.as_ref().clone(); - consumer_plan.site_id = "consumer".to_string(); - consumer_plan.router_identity_id = "/site/consumer/router".to_string(); - consumer_plan.peer_router_identities = BTreeMap::from([( - "authority".to_string(), - MeshIdentityPublic { - id: "/site/authority/router".to_string(), - public_key: [9u8; 32], - mesh_scope: Some("test-mesh".to_string()), - }, - )]); - consumer_plan.peer_router_mesh_addrs = - BTreeMap::from([("authority".to_string(), "127.0.0.1:24000".to_string())]); - let consumer_app = ControlStateApp { - controller_plan: Arc::new(consumer_plan), - ..app +async fn refresh_capability_instances_rejects_unlowered_framework_component_bindings() { + let (_dir, mut state, _state_path, _) = compile_framework_binding_control_state().await; + let binding = state + .base_scenario + .bindings + .iter_mut() + .find(|binding| binding.to.slot == "ctl") + .expect("framework binding should exist"); + binding.from = BindingFromIr::Framework { + authority_realm: state.base_scenario.root, + capability: "component".to_string(), }; - let overlay = framework_route_overlay_payload(&consumer_app) - .await - .expect("consumer framework routes should materialize") - .expect("consumer site should get a framework route overlay"); - + let err = refresh_capability_instances(&mut state) + .expect_err("unlowered framework.component bindings must be rejected"); assert!( - overlay - .peers - .iter() - .any(|peer| peer.id == "/site/authority/router" && peer.public_key == [9u8; 32]), - "consumer overlay should use the planned authority router identity without a peer \ - controller round trip", + err.to_string() + .contains("must be lowered before capability refresh"), + "unexpected framework lowering error: {err}", ); } - #[tokio::test] async fn recover_control_state_reconciles_framework_routes_without_live_peer_controllers() { let (dir, mut state, state_path, _) = compile_framework_binding_control_state().await; @@ -3635,6 +3526,7 @@ async fn recover_control_state_reconciles_framework_routes_without_live_peer_con SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ( @@ -3642,6 +3534,7 @@ async fn recover_control_state_reconciles_framework_routes_without_live_peer_con SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ]); @@ -3690,26 +3583,63 @@ fn inject_site_controller_peer_router_routes_records_peer_router_identities() { version: amber_mesh::MESH_PROVISION_PLAN_VERSION.to_string(), identity_seed: None, existing_peer_identities: Vec::new(), - targets: vec![amber_mesh::MeshProvisionTarget { - kind: amber_mesh::MeshProvisionTargetKind::Router, - config: amber_mesh::MeshConfigTemplate { - identity: amber_mesh::MeshIdentityTemplate { - id: "/site/local/router".to_string(), - mesh_scope: Some("test-mesh".to_string()), + targets: vec![ + amber_mesh::MeshProvisionTarget { + kind: amber_mesh::MeshProvisionTargetKind::Component, + config: amber_mesh::MeshConfigTemplate { + identity: amber_mesh::MeshIdentityTemplate { + id: "/site/local/controller".to_string(), + mesh_scope: Some("test-mesh".to_string()), + }, + mesh_listen: "127.0.0.1:23001".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: vec![amber_mesh::InboundRoute { + route_id: amber_mesh::component_route_id( + "/site/local/controller", + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + MeshProtocol::Http, + ), + capability: + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + .to_string(), + capability_kind: None, + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: amber_mesh::InboundTarget::Local { port: 8080 }, + allowed_issuers: vec!["/site/local/router".to_string()], + }], + outbound: Vec::new(), + transport: TransportConfig::NoiseIk {}, + }, + output: amber_mesh::MeshProvisionOutput::Filesystem { + dir: "mesh/controller".to_string(), }, - mesh_listen: "127.0.0.1:24000".parse().expect("mesh listen"), - control_listen: None, - dynamic_caps_listen: None, - control_allow: None, - peers: Vec::new(), - inbound: Vec::new(), - outbound: Vec::new(), - transport: TransportConfig::NoiseIk {}, }, - output: amber_mesh::MeshProvisionOutput::Filesystem { - dir: "mesh/router".to_string(), + amber_mesh::MeshProvisionTarget { + kind: amber_mesh::MeshProvisionTargetKind::Router, + config: amber_mesh::MeshConfigTemplate { + identity: amber_mesh::MeshIdentityTemplate { + id: "/site/local/router".to_string(), + mesh_scope: Some("test-mesh".to_string()), + }, + mesh_listen: "127.0.0.1:24000".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: TransportConfig::NoiseIk {}, + }, + output: amber_mesh::MeshProvisionOutput::Filesystem { + dir: "mesh/router".to_string(), + }, }, - }], + ], }, ) .expect("mesh provision plan should write"); @@ -3749,6 +3679,37 @@ fn inject_site_controller_peer_router_routes_records_peer_router_identities() { .iter() .find(|target| matches!(target.kind, amber_mesh::MeshProvisionTargetKind::Router)) .expect("router target should remain present"); + assert!( + router.config.inbound.iter().any(|route| { + route.route_id == site_controller_internal_route_id("local") + && route.capability == SITE_CONTROLLER_INTERNAL_CAPABILITY + && matches!( + route.target, + amber_mesh::InboundTarget::MeshForward { + ref peer_id, + ref peer_addr, + ref route_id, + ref capability, + } if peer_id == "/site/local/controller" + && peer_addr + == if cfg!(target_os = "linux") { + "10.0.2.2:23001" + } else { + "127.0.0.1:23001" + } + && route_id + == &amber_mesh::component_route_id( + "/site/local/controller", + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + MeshProtocol::Http, + ) + && capability + == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + ) + }), + "the injected router plan should terminate peer site-controller traffic at the lowered \ + controller component", + ); assert!( router .config @@ -3824,7 +3785,7 @@ struct DynamicCapsMcpHarness { _dir: TempDir, client: Client, base_url: String, - auth_token: String, + control_state_auth_token: String, handles: Vec>, } @@ -3847,7 +3808,6 @@ impl DynamicCapsMcpHarness { let mut handles = vec![install_dynamic_caps_origin_fixture(&app).await]; let controller_app = SiteControllerApp { control: app, - router_auth_token: Arc::::from("test-router-auth"), ready: ready_site_controller_flag(), }; handles.push(tokio::spawn(async move { @@ -3862,20 +3822,44 @@ impl DynamicCapsMcpHarness { _dir: dir, client: Client::new(), base_url, - auth_token: "test-control-state-auth".to_string(), + control_state_auth_token: "test-control-state-auth".to_string(), handles, } } - fn http_headers(&self) -> Vec<(String, String)> { - vec![(FRAMEWORK_AUTH_HEADER.to_string(), self.auth_token.clone())] + fn mcp_headers(&self) -> Vec<(String, String)> { + vec![( + CONTROL_STATE_AUTH_HEADER.to_string(), + self.control_state_auth_token.clone(), + )] + } + + fn dynamic_caps_http_headers(&self, body: &Req) -> Vec<(String, String)> { + let body = serde_json::to_value(body).expect("dynamic caps request should serialize"); + let component_id = body + .get("holder_component_id") + .or_else(|| body.get("caller_component_id")) + .and_then(Value::as_str) + .expect("dynamic caps control requests should identify the caller component"); + let peer_id = dynamic_caps::moniker_from_logical_component_id(component_id) + .expect("logical component id should map to a moniker") + .to_string(); + let route_id = amber_mesh::component_route_id( + peer_id.as_str(), + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + amber_mesh::MeshProtocol::Http, + ); + vec![ + (FRAMEWORK_ROUTE_ID_HEADER.to_string(), route_id), + (FRAMEWORK_PEER_ID_HEADER.to_string(), peer_id), + ] } async fn connect(&self) -> TestMcpClient { TestMcpClient::connect_endpoint( &format!("{}/v1/controller/dynamic-caps/mcp", self.base_url), "framework-dynamic-caps-test", - self.http_headers(), + self.mcp_headers(), ) .await } @@ -3884,7 +3868,7 @@ impl DynamicCapsMcpHarness { http_post_json( &self.client, &format!("{}{}", self.base_url, path), - &self.http_headers(), + &self.dynamic_caps_http_headers(body), body, ) .await @@ -3938,7 +3922,6 @@ async fn install_dynamic_caps_origin_fixture(app: &ControlStateApp) -> tokio::ta "router_mesh_addr": "127.0.0.1:39001", "router_identity_id": plan.router_identity_id, "router_public_key_b64": "dGVzdC1yb3V0ZXIta2V5", - "site_controller_pid": 1, "site_controller_url": plan.authority_url, }), ) @@ -4078,6 +4061,7 @@ async fn dynamic_caps_held_list_ignores_unrouted_offered_sites() { SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, ); state.placement.offered_sites.insert( @@ -4085,13 +4069,13 @@ async fn dynamic_caps_held_list_ignores_unrouted_offered_sites() { SiteDefinition { kind: SiteKind::Vm, context: None, + controller_site: None, }, ); let state_path = dir.path().join("control-state.json"); write_control_state(&state_path, &state).expect("state should write"); let controller_app = SiteControllerApp { control: test_control_state_app(&dir, state, state_path), - router_auth_token: Arc::::from("test-router-auth"), ready: ready_site_controller_flag(), }; @@ -4130,6 +4114,7 @@ async fn localize_framework_control_state_tracks_remote_grant_authority_sites() SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ( @@ -4137,6 +4122,7 @@ async fn localize_framework_control_state_tracks_remote_grant_authority_sites() SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ]), @@ -4206,6 +4192,7 @@ async fn inspect_ref_routes_remote_grants_via_synced_authority_site() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ( @@ -4213,6 +4200,7 @@ async fn inspect_ref_routes_remote_grants_via_synced_authority_site() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ]), @@ -4309,7 +4297,6 @@ async fn inspect_ref_routes_remote_grants_via_synced_authority_site() { BTreeMap::from([("direct_a".to_string(), authority_base_url)]); let controller_app = SiteControllerApp { control: app, - router_auth_token: Arc::::from("test-router-auth"), ready: ready_site_controller_flag(), }; @@ -4354,6 +4341,7 @@ async fn held_list_aggregates_remote_grants_via_peer_router() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ( @@ -4361,6 +4349,7 @@ async fn held_list_aggregates_remote_grants_via_peer_router() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ]), @@ -4460,7 +4449,6 @@ async fn held_list_aggregates_remote_grants_via_peer_router() { BTreeMap::from([("direct_a".to_string(), authority_base_url)]); let controller_app = SiteControllerApp { control: app, - router_auth_token: Arc::::from("test-router-auth"), ready: ready_site_controller_flag(), }; @@ -4950,6 +4938,7 @@ async fn create_snapshot_and_destroy_exact_child() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ( @@ -4957,6 +4946,7 @@ async fn create_snapshot_and_destroy_exact_child() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ]), @@ -5813,6 +5803,7 @@ async fn delegated_cross_site_framework_requests_route_to_the_forwarded_authorit SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ( @@ -5820,6 +5811,7 @@ async fn delegated_cross_site_framework_requests_route_to_the_forwarded_authorit SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ]), @@ -5873,9 +5865,7 @@ async fn delegated_cross_site_framework_requests_route_to_the_forwarded_authorit .cloned() .expect("delegate should have a forwarded framework capability instance"); assert_eq!( - framework_authority_site_id(&direct_state, &direct_record) - .expect("authority site should resolve through composite placement"), - "compose_local", + direct_record.controller_site_id, "compose_local", "delegated root authority should stay anchored at the forwarded realm site", ); let router_public_key_b64 = base64::engine::general_purpose::STANDARD.encode([11u8; 32]); @@ -5927,12 +5917,8 @@ async fn delegated_cross_site_framework_requests_route_to_the_forwarded_authorit dir.path().join("direct-state.json"), ) }, - router_auth_token: Arc::::from("test-router-auth"), ready: ready_site_controller_flag(), }; - reconcile_local_framework_routes(&direct_controller_app.control) - .await - .expect("direct-local controller should publish delegated framework routes"); let stale_err = match super::site_controller::execute_site_controller_framework_mutate( &direct_controller_app, &direct_record, @@ -5952,10 +5938,7 @@ async fn delegated_cross_site_framework_requests_route_to_the_forwarded_authorit }; assert_eq!(stale_err.0.code, ProtocolErrorCode::ControlStateUnavailable); assert!( - stale_err - .0 - .message - .contains("router framework route overlays are stale"), + stale_err.0.message.contains("bound to controller site"), "unexpected stale-delivery error: {}", stale_err.0.message ); @@ -5988,7 +5971,6 @@ async fn delegated_cross_site_framework_requests_route_to_the_forwarded_authorit controller_plan: Arc::new(compose_plan), ..test_control_state_app(&dir, compose_state.clone(), compose_state_path) }, - router_auth_token: Arc::::from("test-router-auth"), ready: ready_site_controller_flag(), }; let response = super::site_controller::execute_site_controller_framework_mutate( @@ -6124,6 +6106,7 @@ async fn create_child_publishes_sites_before_resolving_cross_site_link_overlays( SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, ), ( @@ -6131,6 +6114,7 @@ async fn create_child_publishes_sites_before_resolving_cross_site_link_overlays( SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ]), @@ -6359,7 +6343,7 @@ async fn create_child_publishes_sites_before_resolving_cross_site_link_overlays( } #[tokio::test] -async fn destroy_and_recreate_same_child_name_gets_a_new_capability_instance_id() { +async fn destroy_and_recreate_same_child_name_revokes_then_restores_stable_framework_route() { let dir = TempDir::new().expect("temp dir"); let root_path = dir.path().join("root.json5"); let parent_path = dir.path().join("parent.json5"); @@ -6440,6 +6424,13 @@ async fn destroy_and_recreate_same_child_name_gets_a_new_capability_instance_id( .map(|(id, _)| id.0) .expect("parent component should exist"); + let parent_record = state + .capability_instances + .values() + .find(|record| record.recipient_component_moniker == "/parent") + .cloned() + .expect("parent should have its own framework capability instance"); + create_child( &mut state, parent_id, @@ -6454,12 +6445,30 @@ async fn destroy_and_recreate_same_child_name_gets_a_new_capability_instance_id( ) .await .expect("first delegate create should succeed"); - let first_cap_instance_id = state + authorize_capability_instance(&state, &parent_record.cap_instance_id, "/parent").expect( + "parent authorization should survive when a child receives the same controller route", + ); + let (first_cap_instance_id, first_route_id, first_peer_id) = state .capability_instances .values() .find(|record| record.recipient_component_moniker == "/parent/delegate") - .map(|record| record.cap_instance_id.clone()) + .map(|record| { + ( + record.cap_instance_id.clone(), + record.route_id.clone(), + record.recipient_peer_id.clone(), + ) + }) .expect("first delegate capability instance should exist"); + assert_ne!( + first_cap_instance_id, parent_record.cap_instance_id, + "distinct recipients must retain distinct framework grant identities even when they share \ + a controller route", + ); + assert_eq!( + first_route_id, parent_record.route_id, + "distinct recipients in the same authority should share the same stable controller route", + ); destroy_child(&mut state, parent_id, "delegate", &state_path) .await @@ -6471,6 +6480,10 @@ async fn destroy_and_recreate_same_child_name_gets_a_new_capability_instance_id( .any(|record| record.recipient_component_moniker == "/parent/delegate"), "destroy should revoke the first child lifetime's capability instance", ); + let destroyed = authorize_capability_instance(&state, &first_cap_instance_id, &first_peer_id) + .expect_err("destroyed child should no longer authorize"); + assert_eq!(destroyed.code, ProtocolErrorCode::Unauthorized); + assert_eq!(destroyed.message, "unknown framework capability instance"); create_child( &mut state, @@ -6486,42 +6499,129 @@ async fn destroy_and_recreate_same_child_name_gets_a_new_capability_instance_id( ) .await .expect("second delegate create should succeed"); - let second_cap_instance_id = state + let (second_cap_instance_id, second_route_id, second_peer_id) = state .capability_instances .values() .find(|record| record.recipient_component_moniker == "/parent/delegate") - .map(|record| record.cap_instance_id.clone()) + .map(|record| { + ( + record.cap_instance_id.clone(), + record.route_id.clone(), + record.recipient_peer_id.clone(), + ) + }) .expect("second delegate capability instance should exist"); assert_ne!( - first_cap_instance_id, second_cap_instance_id, - "recreating the same child name must mint a new framework capability instance id", + second_cap_instance_id, first_cap_instance_id, + "recreating the same child name must mint a fresh framework grant identity", + ); + assert_eq!( + second_route_id, first_route_id, + "ordinary framework.component bindings should reuse the stable controller route for the \ + same authority and child path", ); + assert_eq!( + second_peer_id, first_peer_id, + "recreating the same child name should restore the same peer identity path", + ); + authorize_capability_instance(&state, &second_cap_instance_id, &second_peer_id) + .expect("recreated child should regain framework authorization"); } #[test] -fn framework_auth_header_must_match_expected_token() { +fn control_state_auth_header_must_match_expected_token() { let mut headers = HeaderMap::new(); - let missing = authorize_framework_auth_header(&headers, "expected") + let missing = authorize_control_state_auth_header(&headers, "expected") .expect_err("missing auth header should be rejected"); assert_eq!(missing.0.code, ProtocolErrorCode::Unauthorized); headers.insert( - FRAMEWORK_AUTH_HEADER, + CONTROL_STATE_AUTH_HEADER, "wrong".parse().expect("header should parse"), ); - let wrong = authorize_framework_auth_header(&headers, "expected") + let wrong = authorize_control_state_auth_header(&headers, "expected") .expect_err("mismatched auth header should be rejected"); assert_eq!(wrong.0.code, ProtocolErrorCode::Unauthorized); headers.insert( - FRAMEWORK_AUTH_HEADER, + CONTROL_STATE_AUTH_HEADER, "expected".parse().expect("header should parse"), ); - authorize_framework_auth_header(&headers, "expected") + authorize_control_state_auth_header(&headers, "expected") .expect("matching auth header should succeed"); } +#[tokio::test] +async fn local_controller_requests_require_internal_route_headers_not_framework_auth() { + let (dir, state, state_path, _) = compile_framework_binding_control_state().await; + let app = SiteControllerApp { + control: test_control_state_app(&dir, state, state_path), + ready: ready_site_controller_flag(), + }; + let mut headers = HeaderMap::new(); + headers.insert( + super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER, + "1".parse().expect("header should parse"), + ); + headers.insert( + FRAMEWORK_ROUTE_ID_HEADER, + site_controller_internal_route_id(&app.control.controller_plan.site_id) + .parse() + .expect("route header should parse"), + ); + headers.insert( + FRAMEWORK_PEER_ID_HEADER, + app.control + .controller_plan + .router_identity_id + .parse() + .expect("peer header should parse"), + ); + + super::site_controller::authorize_local_controller_request(&app, &headers).expect( + "router-local controller requests should authorize without a control-state auth header", + ); + + headers.insert( + FRAMEWORK_ROUTE_ID_HEADER, + "component:/wrong:__amber_internal_site_controller:http" + .parse() + .expect("header should parse"), + ); + let err = super::site_controller::authorize_local_controller_request(&app, &headers) + .expect_err("wrong internal route should be rejected"); + assert_eq!(err.0.code, ProtocolErrorCode::Unauthorized); +} + +#[tokio::test] +async fn dynamic_caps_sidecar_requests_reject_ambient_control_state_auth_without_internal_route() { + let dir = TempDir::new().expect("temp dir"); + let state = compile_dynamic_caps_binding_state().await; + let state_path = dir.path().join("control-state.json"); + write_control_state(&state_path, &state).expect("state should write"); + let app = SiteControllerApp { + control: test_control_state_app(&dir, state, state_path), + ready: ready_site_controller_flag(), + }; + let mut headers = HeaderMap::new(); + headers.insert( + CONTROL_STATE_AUTH_HEADER, + app.control + .control_state_auth_token + .parse() + .expect("auth header should parse"), + ); + + let err = super::site_controller::authorize_dynamic_caps_sidecar_request( + &app, + &headers, + "components./alice", + ) + .expect_err("ambient control-state auth must not bypass the controller internal route"); + assert_eq!(err.0.code, ProtocolErrorCode::Unauthorized); +} + #[tokio::test] async fn dynamic_authority_templates_are_listed_and_created_from_live_realm() { let dir = TempDir::new().expect("temp dir"); @@ -7020,6 +7120,7 @@ async fn snapshot_is_stable_across_dynamic_create_order() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ( @@ -7027,6 +7128,7 @@ async fn snapshot_is_stable_across_dynamic_create_order() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ]), @@ -7159,6 +7261,7 @@ async fn create_rejects_unoffered_backend_without_committing_child_state() { SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -7278,6 +7381,7 @@ async fn concurrent_distinct_creates_commit_both_children() { SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -7402,6 +7506,7 @@ async fn prepare_child_record_uses_frozen_dynamic_placement_assignments() { SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, ), ( @@ -7409,6 +7514,7 @@ async fn prepare_child_record_uses_frozen_dynamic_placement_assignments() { SiteDefinition { kind: SiteKind::Kubernetes, context: None, + controller_site: None, }, ), ]), @@ -7606,6 +7712,7 @@ async fn prepare_child_record_supports_cross_site_dynamic_fragments() { SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, ), ( @@ -7613,6 +7720,7 @@ async fn prepare_child_record_supports_cross_site_dynamic_fragments() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, ), ( @@ -7620,6 +7728,7 @@ async fn prepare_child_record_supports_cross_site_dynamic_fragments() { SiteDefinition { kind: SiteKind::Kubernetes, context: None, + controller_site: None, }, ), ( @@ -7627,6 +7736,7 @@ async fn prepare_child_record_supports_cross_site_dynamic_fragments() { SiteDefinition { kind: SiteKind::Vm, context: None, + controller_site: None, }, ), ]), @@ -8283,6 +8393,7 @@ async fn recover_control_state_surfaces_create_prepared_rollback_failures() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -8394,6 +8505,7 @@ async fn recover_control_state_does_not_republish_live_children() { SiteDefinition { kind: SiteKind::Direct, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults { @@ -9112,6 +9224,13 @@ services: image: busybox amber-provisioner: image: ghcr.io/rdi-foundation/amber-provisioner:test + amber-site-controller: + image: __amber_internal/site-controller + command: + - --plan + - /amber/site/state/site-controller-plan.json + environment: + - EXISTING_ENV=kept networks: amber_mesh: {} volumes: @@ -9120,14 +9239,15 @@ volumes: ) .expect("compose yaml should write"); + let controller_port = 32111; let plan = write_site_controller_plan( &site_controller_plan_path(&site_state_root), "test-run", "test-mesh", "compose-site", SiteKind::Compose, - SocketAddr::from(([0, 0, 0, 0], SITE_CONTROLLER_PORT)), - &format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{SITE_CONTROLLER_PORT}"), + SocketAddr::from(([0, 0, 0, 0], controller_port)), + &format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{controller_port}"), "/site/compose-site/router", &BTreeMap::new(), &BTreeMap::new(), @@ -9179,11 +9299,11 @@ volumes: .get(serde_yaml::Value::String("command".to_string())) .and_then(serde_yaml::Value::as_sequence) .expect("controller service should have a command"); - let plan_path = site_controller_plan_path(&site_state_root) - .display() - .to_string(); assert_eq!(command[0].as_str(), Some("--plan")); - assert_eq!(command[1].as_str(), Some(plan_path.as_str())); + assert_eq!( + command[1].as_str(), + Some("/amber/site/state/site-controller-plan.json") + ); assert_eq!( service .get(serde_yaml::Value::String("user".to_string())) @@ -9212,6 +9332,15 @@ volumes: temp.path().display() )) })); + let environment = service + .get(serde_yaml::Value::String("environment".to_string())) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller service should preserve environment"); + assert!( + environment + .iter() + .any(|value| value.as_str() == Some("EXISTING_ENV=kept")) + ); assert!( volumes .iter() @@ -9258,7 +9387,8 @@ fn kubernetes_site_controller_resources_are_injected_into_the_artifact() { fs::create_dir_all(site_state_root.clone()).expect("site state root should create"); fs::write( artifact_root.join("kustomization.yaml"), - "resources:\n - 05-networkpolicies/amber-router-netpol.yaml\n", + "resources:\n - 03-deployments/amber-site-controller.yaml\n - \ + 04-services/amber-site-controller.yaml\n - 05-networkpolicies/amber-router-netpol.yaml\n", ) .expect("kustomization should write"); fs::write( @@ -9308,14 +9438,57 @@ spec: "http://amber-router:37046".to_string(), )]); + fs::create_dir_all(artifact_root.join("03-deployments")) + .expect("deployments dir should create"); + fs::write( + artifact_root.join("03-deployments/amber-site-controller.yaml"), + r#" +apiVersion: apps/v1 +kind: Deployment +metadata: + name: amber-site-controller +spec: + template: + spec: + containers: + - name: main + image: __amber_internal/site-controller + args: + - --plan + - /amber/site/state/site-controller-plan.json + env: + - name: EXISTING_ENV + value: kept + - name: sidecar + image: ghcr.io/rdi-foundation/amber-router:test +"#, + ) + .expect("controller deployment should write"); + fs::write( + artifact_root.join("04-services/amber-site-controller.yaml"), + r#" +apiVersion: v1 +kind: Service +metadata: + name: amber-site-controller +spec: + ports: + - name: framework-component + port: 32123 + targetPort: 32123 + protocol: TCP +"#, + ) + .expect("controller service should write"); + let controller_port = 32123; let plan = write_site_controller_plan( &site_controller_plan_path(&site_state_root), "test-run", "test-mesh", "kube-site", SiteKind::Kubernetes, - SocketAddr::from(([0, 0, 0, 0], SITE_CONTROLLER_PORT)), - &format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{SITE_CONTROLLER_PORT}"), + SocketAddr::from(([0, 0, 0, 0], controller_port)), + &format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{controller_port}"), "/site/kube-site/router", &peer_site_router_urls, &BTreeMap::new(), @@ -9375,11 +9548,26 @@ spec: "kubernetes site controller seeding should reuse the site-controller image so tests and \ CI do not depend on an extra public init image pull", ); - let container = deployment["spec"]["template"]["spec"]["containers"] + let containers = deployment["spec"]["template"]["spec"]["containers"] .as_sequence() - .and_then(|containers| containers.first()) + .expect("deployment should contain site-controller containers"); + assert_eq!( + containers.len(), + 2, + "kubernetes bootstrap should patch the rendered controller deployment instead of \ + replacing it" + ); + let container = containers + .iter() + .find(|container| { + container + .as_mapping() + .and_then(|mapping| mapping.get(serde_yaml::Value::String("name".to_string()))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|name| name == "main") + }) .and_then(serde_yaml::Value::as_mapping) - .expect("deployment should contain a site-controller container"); + .expect("deployment should keep the main site-controller container"); assert!( !container.contains_key(serde_yaml::Value::String("command".to_string())), "kubernetes site controller should keep the image entrypoint and pass only args" @@ -9399,7 +9587,8 @@ spec: .expect("seed configmap should read"); assert!(seed.contains("site-controller-plan.json")); assert!(seed.contains("artifact.tar.b64")); - assert!(seed.contains("http://amber-site-controller:4100")); + assert!(seed.contains(&format!("http://amber-site-controller:{controller_port}"))); + assert!(deployment_raw.contains("EXISTING_ENV")); let router_netpol = fs::read_to_string(artifact_root.join("05-networkpolicies/amber-router-netpol.yaml")) @@ -9439,6 +9628,7 @@ fn local_site_manager_state_uses_controller_plan_when_host_state_is_absent() { SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults::default(), @@ -9481,8 +9671,7 @@ fn local_site_manager_state_uses_controller_plan_when_host_state_is_absent() { controller_plan.local_router_control = Some("unix:///amber/control/router-control.sock".to_string()); controller_plan.published_router_mesh_addr = Some("127.0.0.1:24000".to_string()); - controller_plan.authority_url = - format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{SITE_CONTROLLER_PORT}"); + controller_plan.authority_url = "http://amber-site-controller:32111".to_string(); let state = load_site_manager_state(&app, "compose-site") .expect("local controller should synthesize site metadata from its own plan"); @@ -9492,7 +9681,7 @@ fn local_site_manager_state_uses_controller_plan_when_host_state_is_absent() { Some("unix:///amber/control/router-control.sock") ); assert_eq!(state.router_mesh_addr.as_deref(), Some("127.0.0.1:24000")); - let authority_url = format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{SITE_CONTROLLER_PORT}"); + let authority_url = "http://amber-site-controller:32111".to_string(); assert_eq!( state.site_controller_url.as_deref(), Some(authority_url.as_str()) @@ -9525,6 +9714,7 @@ fn load_site_manager_state_prefers_local_controller_view_over_stale_host_state() SiteDefinition { kind: SiteKind::Compose, context: None, + controller_site: None, }, )]), defaults: PlacementDefaults::default(), @@ -9580,8 +9770,7 @@ fn load_site_manager_state_prefers_local_controller_view_over_stale_host_state() controller_plan.local_router_control = Some("unix:///amber/control/router-control.sock".to_string()); controller_plan.published_router_mesh_addr = Some("127.0.0.1:24000".to_string()); - controller_plan.authority_url = - format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{SITE_CONTROLLER_PORT}"); + controller_plan.authority_url = "http://amber-site-controller:32111".to_string(); let state = load_site_manager_state(&app, "compose-site") .expect("local controller should not trust stale host manager state"); @@ -9591,7 +9780,7 @@ fn load_site_manager_state_prefers_local_controller_view_over_stale_host_state() Some("unix:///amber/control/router-control.sock") ); assert_eq!(state.router_mesh_addr.as_deref(), Some("127.0.0.1:24000")); - let authority_url = format!("http://{SITE_CONTROLLER_SERVICE_NAME}:{SITE_CONTROLLER_PORT}"); + let authority_url = "http://amber-site-controller:32111".to_string(); assert_eq!( state.site_controller_url.as_deref(), Some(authority_url.as_str()) From df5fe0c9bca5c1471256331018925ece32c1773a Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Fri, 1 May 2026 17:25:59 +0800 Subject: [PATCH 11/25] Fix more tests --- cli/src/main.rs | 16 +- cli/src/mixed_run.rs | 7 + cli/src/mixed_run/supervisor.rs | 2 +- cli/src/mixed_run/tests.rs | 60 +- cli/tests/compile_outputs/artifact_outputs.rs | 711 +--------- cli/tests/compile_outputs/mod.rs | 5 +- cli/tests/mixed_run/dry_run.rs | 63 +- cli/tests/mixed_run/smoke.rs | 62 +- cli/tests/mixed_run/support.rs | 66 + compiler/src/run_plan.rs | 926 ++++++++++++- compiler/src/targets/direct/mod.rs | 31 +- .../src/targets/mesh/docker_compose/mod.rs | 2 +- .../mesh/docker_compose/tests/rendering.rs | 7 + compiler/src/targets/mesh/kubernetes/mod.rs | 13 +- .../src/targets/mesh/kubernetes/tests/mod.rs | 39 +- .../mesh/kubernetes/tests/rendering.rs | 56 +- compiler/src/targets/mesh/mesh_config.rs | 39 +- runtime/mesh/src/lib.rs | Bin 20380 -> 21670 bytes runtime/router/src/control.rs | 2 +- runtime/router/src/dynamic_caps.rs | 44 +- .../site-controller/src/control_state_api.rs | 6 +- .../site-controller/src/default_runtime.rs | 13 +- .../src/default_runtime/child_runtime.rs | 425 +++++- .../src/default_runtime/compose_controller.rs | 83 +- .../src/default_runtime/dynamic_routes.rs | 1208 +++++++++++------ .../default_runtime/kubernetes_controller.rs | 541 +++++++- .../src/default_runtime/site_artifacts.rs | 38 +- runtime/site-controller/src/lib.rs | 10 +- runtime/site-controller/src/orchestration.rs | 456 ++++--- runtime/site-controller/src/planner.rs | 430 ++++-- runtime/site-controller/src/runtime_api.rs | 4 + .../site-controller/src/site_controller.rs | 376 ++--- runtime/site-controller/src/state.rs | 16 +- runtime/site-controller/src/tests.rs | 905 ++++++++++-- 34 files changed, 4659 insertions(+), 2003 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 007b05d5..4e3f7933 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -39,7 +39,7 @@ use amber_compiler::{ dot::DotReporter, metadata::MetadataReporter, scenario_ir::ScenarioIrReporter, - vm::VM_PLAN_FILENAME, + vm::{VM_PLAN_FILENAME, VM_RUN_SCRIPT_FILENAME, VmReporter}, }, run_plan::{ PlacementFile, RUN_PLAN_SCHEMA, RunPlan, SiteKind, build_homogeneous_export_run_plan, @@ -1154,10 +1154,16 @@ async fn compile(args: CompileArgs) -> Result<()> { } if let Some(vm_dest) = outputs.vm { - let run_plan = build_homogeneous_export_run_plan(&compiled, SiteKind::Vm) - .into_diagnostic() - .wrap_err("failed to build homogeneous vm export plan")?; - write_unmanaged_export_output(&vm_dest, &run_plan, SiteKind::Vm)?; + let artifact = VmReporter + .emit(&compiled) + .map_err(miette::Report::new) + .wrap_err("failed to build vm artifact")?; + write_directory_output( + &vm_dest, + "vm output directory", + &artifact.files, + Some(Path::new(VM_RUN_SCRIPT_FILENAME)), + )?; } if let Some(metadata_dest) = outputs.metadata { diff --git a/cli/src/mixed_run.rs b/cli/src/mixed_run.rs index d404a3ca..c96a2fea 100644 --- a/cli/src/mixed_run.rs +++ b/cli/src/mixed_run.rs @@ -130,6 +130,13 @@ fn site_controller_command_from(current: &Path) -> Result prefix_args: Vec::new(), }); } + let amber_executable_name = format!("amber{}", std::env::consts::EXE_SUFFIX); + if current.file_name().and_then(|name| name.to_str()) == Some(amber_executable_name.as_str()) { + return Ok(SiteControllerCommand { + executable: current.to_path_buf(), + prefix_args: vec!["run-site-controller"], + }); + } if launched_from_cargo_test_binary(current) { return Ok(SiteControllerCommand { executable: amber_cli_executable_from(current)?, diff --git a/cli/src/mixed_run/supervisor.rs b/cli/src/mixed_run/supervisor.rs index ac5200ab..41b2f824 100644 --- a/cli/src/mixed_run/supervisor.rs +++ b/cli/src/mixed_run/supervisor.rs @@ -935,7 +935,7 @@ pub(super) fn local_site_controller_ready( timeout: Duration, ) -> Result { let Some(addr) = local_site_controller_addr(plan)? else { - return Ok(false); + return Ok(true); }; Ok(wait_for_http_response(addr, timeout).is_ok()) } diff --git a/cli/src/mixed_run/tests.rs b/cli/src/mixed_run/tests.rs index dece2640..bc49db92 100644 --- a/cli/src/mixed_run/tests.rs +++ b/cli/src/mixed_run/tests.rs @@ -94,11 +94,6 @@ fn site_controller_local_router_control_uses_backend_local_control_targets() { .starts_with("unix://"), "direct site controllers should know their local router control socket up front" ); - assert!( - launch_bundle::site_controller_local_router_control(SiteKind::Vm, artifact_dir) - .starts_with("unix://"), - "vm site controllers should know their local router control socket up front" - ); } #[test] @@ -191,6 +186,28 @@ fn provisioner_image_supports_debug_builds() { ); } +#[test] +fn site_controller_command_prefers_current_amber_binary_over_stale_sibling() { + let temp = TempDir::new().expect("temp dir"); + let debug_dir = temp.path().join("target").join("debug"); + fs::create_dir_all(&debug_dir).expect("debug dir should exist"); + let amber = debug_dir.join(format!("amber{}", std::env::consts::EXE_SUFFIX)); + let site_controller = debug_dir.join(format!( + "amber-site-controller{}", + std::env::consts::EXE_SUFFIX + )); + fs::write(&amber, "").expect("amber binary should exist"); + fs::write(&site_controller, "").expect("stale site controller binary should exist"); + + let command = site_controller_command_from(&amber).expect("site controller command"); + assert_eq!( + command.executable, amber, + "the running amber binary should launch its own run-site-controller subcommand instead of \ + a stale sibling amber-site-controller executable" + ); + assert_eq!(command.prefix_args, vec!["run-site-controller"]); +} + #[test] fn site_controller_command_prefers_fresh_amber_binary_under_cargo_tests() { let temp = TempDir::new().expect("temp dir"); @@ -216,6 +233,39 @@ fn site_controller_command_prefers_fresh_amber_binary_under_cargo_tests() { assert_eq!(command.prefix_args, vec!["run-site-controller"]); } +#[test] +fn local_site_controller_ready_treats_absent_controller_as_optional() { + let plan = SiteSupervisorPlan { + schema: "amber.run.site_supervisor_plan".to_string(), + version: 2, + run_id: "run".to_string(), + mesh_scope: "scope".to_string(), + run_root: "/tmp/run".to_string(), + coordinator_pid: 1, + site_id: "vm-local".to_string(), + kind: SiteKind::Vm, + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), + storage_root: None, + runtime_root: None, + router_mesh_port: None, + compose_project: None, + kubernetes_namespace: None, + context: None, + port_forward_mesh_port: None, + port_forward_control_port: None, + observability_endpoint: None, + site_controller_url: None, + launch_env: BTreeMap::new(), + }; + + assert!( + supervisor::local_site_controller_ready(&plan, Duration::from_millis(10)) + .expect("sites without a local controller should still be considered ready"), + "managed vm sites should not block startup on a non-existent local controller" + ); +} + #[test] fn local_site_controller_addr_requires_loopback_http() { let mut plan = SiteSupervisorPlan { diff --git a/cli/tests/compile_outputs/artifact_outputs.rs b/cli/tests/compile_outputs/artifact_outputs.rs index d4ed1dc1..30f1f47e 100644 --- a/cli/tests/compile_outputs/artifact_outputs.rs +++ b/cli/tests/compile_outputs/artifact_outputs.rs @@ -387,9 +387,10 @@ fn compile_writes_direct_artifact() { "direct plan should not persist manifest_url" ); assert!( - components - .iter() - .all(|component| component.get("source_dir").is_some()), + components.iter().all(|component| { + component["program"]["execution"]["kind"].as_str() == Some("internal_site_controller") + || component.get("source_dir").is_some() + }), "manifest-backed direct plan components should retain source_dir" ); assert_eq!( @@ -410,8 +411,21 @@ fn compile_writes_direct_artifact() { assert_eq!(proxy_json["router"]["control_port"].as_u64(), Some(0)); } +fn assert_vm_export_succeeded(output: &std::process::Output) { + assert!( + output.status.success(), + "amber compile --vm failed +stdout: +{} +stderr: +{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + #[test] -fn compile_writes_vm_artifact() { +fn compile_vm_export_uses_default_placement() { let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) .parent() .expect("cli crate should live under the workspace root"); @@ -472,91 +486,21 @@ fn compile_writes_vm_artifact() { .output() .unwrap_or_else(|err| panic!("failed to run amber compile --vm: {err}")); - if !output.status.success() { - panic!( - "amber compile --vm failed\nstatus: {}\nstdout:\n{}\nstderr:\n{}", - output.status, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - } - - let vm_plan_path = artifact_dir.join(VM_PLAN_FILENAME); - let provision_plan_path = artifact_dir.join("mesh-provision-plan.json"); - let run_script_path = artifact_dir.join("run.sh"); - let readme_path = artifact_dir.join("README.md"); - let env_example_path = artifact_dir.join("env.example"); - let proxy_metadata_path = artifact_dir.join("amber-proxy.json"); - - assert!(vm_plan_path.is_file(), "missing vm-plan.json"); - assert!( - provision_plan_path.is_file(), - "missing mesh-provision-plan.json" - ); - assert!(run_script_path.is_file(), "missing run.sh"); - assert!(readme_path.is_file(), "missing README.md"); - assert!(env_example_path.is_file(), "missing env.example"); - assert!(proxy_metadata_path.is_file(), "missing amber-proxy.json"); - - let vm_plan: Value = serde_json::from_str( - &fs::read_to_string(&vm_plan_path).expect("failed to read vm-plan.json"), - ) - .expect("vm-plan.json should be valid JSON"); - assert_eq!(vm_plan["version"], VM_PLAN_VERSION); - assert_eq!(vm_plan["mesh_provision_plan"], "mesh-provision-plan.json"); - let components = vm_plan["components"] - .as_array() - .expect("vm plan components should be an array"); - assert_eq!(components.len(), 1, "{vm_plan:#}"); - assert_eq!(components[0]["base_image"]["kind"], "runtime_config"); - assert_eq!(components[0]["base_image"]["query"], "base_image"); - assert_eq!(components[0]["cpus"]["kind"], "literal"); - assert_eq!(components[0]["cpus"]["value"], 1); - assert_eq!(components[0]["memory_mib"]["kind"], "literal"); - assert_eq!(components[0]["memory_mib"]["value"], 512); - let component = components[0] - .as_object() - .expect("vm component should serialize as an object"); - let allowed_root_leaf_paths = component["runtime_config"]["allowed_root_leaf_paths"] - .as_array() - .expect("runtime_config.allowed_root_leaf_paths should be an array") - .iter() - .map(|value| value.as_str().expect("allowed path should be a string")) - .collect::>(); + assert_vm_export_succeeded(&output); assert!( - component.contains_key("runtime_config"), - "vm component should carry runtime_config when the base image depends on runtime config" + artifact_dir.join("vm-plan.json").is_file(), + "expected vm-plan.json in {}", + artifact_dir.display() ); - assert_eq!(allowed_root_leaf_paths, vec!["base_image"]); assert!( - !component.contains_key("mount_spec_b64"), - "vm component should omit mount_spec_b64 when it is unused" - ); - - let readme = fs::read_to_string(&readme_path).expect("failed to read vm README"); - assert!( - readme.contains("amber run ."), - "vm README should explain how to start the output" - ); - let env_example = fs::read_to_string(&env_example_path).expect("failed to read vm env example"); - assert!( - env_example.contains("AMBER_CONFIG_BASE_IMAGE"), - "vm env example should mention runtime config for the base image" - ); - - let proxy_metadata: Value = serde_json::from_str( - &fs::read_to_string(&proxy_metadata_path).expect("failed to read vm proxy metadata"), - ) - .expect("vm proxy metadata should be valid JSON"); - assert_eq!( - proxy_metadata["router"]["control_socket"].as_str(), - Some(DIRECT_CONTROL_SOCKET_RELATIVE_PATH) + artifact_dir.join("run.sh").is_file(), + "expected run.sh in {}", + artifact_dir.display() ); - assert_eq!(proxy_metadata["router"]["control_port"].as_u64(), Some(0)); } #[test] -fn compile_vm_artifact_supports_runtime_templates() { +fn compile_vm_export_rejects_framework_component_without_controller_site() { let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) .parent() .expect("cli crate should live under the workspace root"); @@ -564,531 +508,21 @@ fn compile_vm_artifact_supports_runtime_templates() { let outputs_root = workspace_root.join("target").join("cli-test-outputs"); fs::create_dir_all(&outputs_root).expect("failed to create outputs directory"); let outputs_dir = tempfile::Builder::new() - .prefix("vm-runtime-template-") + .prefix("vm-framework-outputs-") .tempdir_in(&outputs_root) .expect("failed to create outputs directory"); - let manifest = outputs_dir.path().join("scenario.json5"); + let admin = outputs_dir.path().join("admin.json5"); fs::write( - &manifest, + &admin, r##"{ - manifest_version: "0.1.0", - config_schema: { - type: "object", - properties: { - base_image: { type: "string" }, - message: { type: "string" } - }, - required: ["base_image", "message"], - additionalProperties: false - }, + manifest_version: "0.3.0", slots: { - api: { kind: "http", optional: true } + ctl: { kind: "component" } }, program: { - vm: { - image: "./images/${config.base_image}", - cpus: 1, - memory_mib: 512, - cloud_init: { - user_data: "#cloud-config\nwrite_files:\n - path: /tmp/value\n content: 'msg=${config.message};api=${slots.api.url}'\n" - }, - network: { - endpoints: [ - { name: "http", port: 8080, protocol: "http" } - ], - egress: "none" - } - } - }, - provides: { - http: { kind: "http", endpoint: "http" } - }, - exports: { - http: "http" - } -} -"##, - ) - .expect("failed to write manifest"); - - let artifact_dir = outputs_dir.path().join("vm"); - let output = Command::new(env!("CARGO_BIN_EXE_amber")) - .arg("compile") - .arg("--vm") - .arg(&artifact_dir) - .arg(&manifest) - .output() - .unwrap_or_else(|err| panic!("failed to run amber compile --vm: {err}")); - - if !output.status.success() { - panic!( - "amber compile --vm failed\nstatus: {}\nstdout:\n{}\nstderr:\n{}", - output.status, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - } - - let vm_plan_path = artifact_dir.join(VM_PLAN_FILENAME); - let vm_plan: Value = serde_json::from_str( - &fs::read_to_string(&vm_plan_path).expect("failed to read vm-plan.json"), - ) - .expect("vm-plan.json should be valid JSON"); - let component = vm_plan["components"] - .as_array() - .and_then(|components| components.first()) - .expect("vm plan should contain one component"); - - assert_eq!(component["base_image"]["kind"], "runtime_template"); - assert_eq!(component["base_image"]["parts"][0]["kind"], "literal"); - assert_eq!(component["base_image"]["parts"][0]["value"], "./images/"); - assert_eq!( - component["base_image"]["parts"][1]["kind"], - "runtime_config" - ); - assert_eq!(component["base_image"]["parts"][1]["query"], "base_image"); - assert_eq!( - component["base_image"]["source_dir"].as_str(), - outputs_dir.path().to_str() - ); - - assert_eq!( - component["cloud_init_user_data"]["kind"], - "runtime_template" - ); - let cloud_init_parts = component["cloud_init_user_data"]["parts"] - .as_array() - .expect("cloud-init template parts should be an array"); - assert!( - cloud_init_parts - .iter() - .any(|part| part["config"] == "message"), - "cloud-init template should retain config interpolation" - ); - assert!( - cloud_init_parts - .iter() - .any(|part| part["slot"] == "api.url"), - "cloud-init template should retain slot interpolation" - ); - let mut allowed_root_leaf_paths = component["runtime_config"]["allowed_root_leaf_paths"] - .as_array() - .expect("runtime_config.allowed_root_leaf_paths should be an array") - .iter() - .map(|value| { - value - .as_str() - .expect("allowed_root_leaf_paths should contain strings") - .to_string() - }) - .collect::>(); - allowed_root_leaf_paths.sort(); - assert_eq!( - allowed_root_leaf_paths, - vec!["base_image".to_string(), "message".to_string()] - ); -} - -#[test] -fn compile_vm_artifact_resolves_static_child_config_for_scalars_and_cloud_init() { - let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .expect("cli crate should live under the workspace root"); - - let outputs_root = workspace_root.join("target").join("cli-test-outputs"); - fs::create_dir_all(&outputs_root).expect("failed to create outputs directory"); - let outputs_dir = tempfile::Builder::new() - .prefix("vm-static-child-config-") - .tempdir_in(&outputs_root) - .expect("failed to create outputs directory"); - - let child_manifest = outputs_dir.path().join("child.json5"); - fs::write( - &child_manifest, - r##"{ - manifest_version: "0.1.0", - config_schema: { - type: "object", - properties: { - image: { type: "string" }, - cpu_count: { type: "integer" }, - memory: { type: "integer" }, - banner: { type: "string" } - }, - required: ["image", "cpu_count", "memory", "banner"], - additionalProperties: false - }, - program: { - vm: { - image: "${config.image}", - cpus: "${config.cpu_count}", - memory_mib: "${config.memory}", - cloud_init: { - user_data: "#cloud-config\nwrite_files:\n - path: /tmp/banner\n content: '${config.banner}'\n" - }, - network: { - endpoints: [ - { name: "http", port: 8080, protocol: "http" } - ], - egress: "none" - } - } - }, - provides: { - http: { kind: "http", endpoint: "http" } - }, - exports: { - http: "http" - } -} -"##, - ) - .expect("failed to write child manifest"); - - let root_manifest = outputs_dir.path().join("scenario.json5"); - fs::write( - &root_manifest, - format!( - r##"{{ - manifest_version: "0.1.0", - components: {{ - child: {{ - manifest: "{}", - config: {{ - image: "./images/base.qcow2", - cpu_count: 3, - memory: 1536, - banner: "hello from parent" - }} - }} - }}, - exports: {{ - http: "#child.http" - }} -}} -"##, - child_manifest.display() - ), - ) - .expect("failed to write root manifest"); - - let artifact_dir = outputs_dir.path().join("vm"); - let output = Command::new(env!("CARGO_BIN_EXE_amber")) - .arg("compile") - .arg("--vm") - .arg(&artifact_dir) - .arg(&root_manifest) - .output() - .unwrap_or_else(|err| panic!("failed to run amber compile --vm: {err}")); - - if !output.status.success() { - panic!( - "amber compile --vm failed\nstatus: {}\nstdout:\n{}\nstderr:\n{}", - output.status, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - } - - let vm_plan: Value = serde_json::from_str( - &fs::read_to_string(artifact_dir.join(VM_PLAN_FILENAME)) - .expect("failed to read vm-plan.json"), - ) - .expect("vm-plan.json should be valid JSON"); - let component = vm_plan["components"] - .as_array() - .and_then(|components| components.first()) - .expect("vm plan should contain one component"); - - assert_eq!(component["base_image"]["kind"], "static"); - assert_eq!(component["cpus"]["kind"], "literal"); - assert_eq!(component["cpus"]["value"], 3); - assert_eq!(component["memory_mib"]["kind"], "literal"); - assert_eq!(component["memory_mib"]["value"], 1536); - assert_eq!(component["cloud_init_user_data"]["kind"], "static"); - assert!( - component["cloud_init_user_data"]["value"] - .as_str() - .is_some_and(|value| value.contains("hello from parent")), - "{component:#}" - ); - let component = component - .as_object() - .expect("vm component should serialize as an object"); - assert!( - !component.contains_key("runtime_config"), - "static child config should not force runtime_config" - ); -} - -#[test] -fn compile_vm_artifact_rewrites_runtime_scalar_paths_through_child_config() { - let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .expect("cli crate should live under the workspace root"); - - let outputs_root = workspace_root.join("target").join("cli-test-outputs"); - fs::create_dir_all(&outputs_root).expect("failed to create outputs directory"); - let outputs_dir = tempfile::Builder::new() - .prefix("vm-runtime-child-config-") - .tempdir_in(&outputs_root) - .expect("failed to create outputs directory"); - - let child_manifest = outputs_dir.path().join("child.json5"); - fs::write( - &child_manifest, - r##"{ - manifest_version: "0.1.0", - config_schema: { - type: "object", - properties: { - image: { type: "string" }, - cpu_count: { type: "integer" }, - memory: { type: "integer" } - }, - required: ["image", "cpu_count", "memory"], - additionalProperties: false - }, - program: { - vm: { - image: "${config.image}", - cpus: "${config.cpu_count}", - memory_mib: "${config.memory}", - network: { - endpoints: [ - { name: "http", port: 8080, protocol: "http" } - ], - egress: "none" - } - } - }, - provides: { - http: { kind: "http", endpoint: "http" } - }, - exports: { - http: "http" - } -} -"##, - ) - .expect("failed to write child manifest"); - - let root_manifest = outputs_dir.path().join("scenario.json5"); - fs::write( - &root_manifest, - format!( - r##"{{ - manifest_version: "0.1.0", - config_schema: {{ - type: "object", - properties: {{ - vm: {{ - type: "object", - properties: {{ - image: {{ type: "string" }}, - cpu: {{ type: "integer" }}, - memory: {{ type: "integer" }} - }}, - required: ["image", "cpu", "memory"], - additionalProperties: false - }} - }}, - required: ["vm"], - additionalProperties: false - }}, - components: {{ - child: {{ - manifest: "{}", - config: {{ - image: "${{config.vm.image}}", - cpu_count: "${{config.vm.cpu}}", - memory: "${{config.vm.memory}}" - }} - }} - }}, - exports: {{ - http: "#child.http" - }} -}} -"##, - child_manifest.display() - ), - ) - .expect("failed to write root manifest"); - - let artifact_dir = outputs_dir.path().join("vm"); - let output = Command::new(env!("CARGO_BIN_EXE_amber")) - .arg("compile") - .arg("--vm") - .arg(&artifact_dir) - .arg(&root_manifest) - .output() - .unwrap_or_else(|err| panic!("failed to run amber compile --vm: {err}")); - - if !output.status.success() { - panic!( - "amber compile --vm failed\nstatus: {}\nstdout:\n{}\nstderr:\n{}", - output.status, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - } - - let vm_plan: Value = serde_json::from_str( - &fs::read_to_string(artifact_dir.join(VM_PLAN_FILENAME)) - .expect("failed to read vm-plan.json"), - ) - .expect("vm-plan.json should be valid JSON"); - let component = vm_plan["components"] - .as_array() - .and_then(|components| components.first()) - .expect("vm plan should contain one component"); - - assert_eq!(component["base_image"]["kind"], "runtime_config"); - assert_eq!(component["base_image"]["query"], "image"); - assert_eq!(component["cpus"]["kind"], "runtime_config"); - assert_eq!(component["cpus"]["query"], "cpu_count"); - assert_eq!(component["memory_mib"]["kind"], "runtime_config"); - assert_eq!(component["memory_mib"]["query"], "memory"); - let mut allowed_root_leaf_paths = component["runtime_config"]["allowed_root_leaf_paths"] - .as_array() - .expect("runtime_config.allowed_root_leaf_paths should be an array") - .iter() - .map(|value| { - value - .as_str() - .expect("allowed_root_leaf_paths should contain strings") - .to_string() - }) - .collect::>(); - allowed_root_leaf_paths.sort(); - assert_eq!( - allowed_root_leaf_paths, - vec![ - "vm.cpu".to_string(), - "vm.image".to_string(), - "vm.memory".to_string(), - ] - ); - assert!( - component.get("runtime_config").is_some(), - "runtime child config should carry runtime_config payload" - ); - let env_example = fs::read_to_string(artifact_dir.join("env.example")) - .expect("failed to read vm env example"); - assert!( - env_example.contains("AMBER_CONFIG_VM__IMAGE"), - "vm env example should mention runtime config for the forwarded base image" - ); - assert!( - env_example.contains("AMBER_CONFIG_VM__CPU"), - "vm env example should mention runtime config for the forwarded cpu count" - ); - assert!( - env_example.contains("AMBER_CONFIG_VM__MEMORY"), - "vm env example should mention runtime config for the forwarded memory size" - ); -} - -#[test] -fn check_rejects_invalid_vm_scalar_config_ref() { - let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .expect("cli crate should live under the workspace root"); - - let outputs_root = workspace_root.join("target").join("cli-test-outputs"); - fs::create_dir_all(&outputs_root).expect("failed to create outputs directory"); - let outputs_dir = tempfile::Builder::new() - .prefix("vm-invalid-scalar-check-") - .tempdir_in(&outputs_root) - .expect("failed to create outputs directory"); - - let manifest = outputs_dir.path().join("scenario.json5"); - fs::write( - &manifest, - r##"{ - manifest_version: "0.1.0", - config_schema: { - type: "object", - properties: { - image: { type: "string" } - }, - required: ["image"], - additionalProperties: false - }, - program: { - vm: { - image: "${config.image}", - cpus: "${config.missing}", - memory_mib: 512, - network: { - endpoints: [ - { name: "http", port: 8080, protocol: "http" } - ], - egress: "none" - } - } - }, - provides: { - http: { kind: "http", endpoint: "http" } - }, - exports: { - http: "http" - } -} -"##, - ) - .expect("failed to write manifest"); - - let output = Command::new(env!("CARGO_BIN_EXE_amber")) - .arg("check") - .arg(&manifest) - .output() - .unwrap_or_else(|err| panic!("failed to run amber check: {err}")); - - assert!( - !output.status.success(), - "amber check unexpectedly succeeded" - ); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!( - stderr.contains("program.vm.cpus"), - "expected program.vm.cpus in stderr, got:\n{stderr}" - ); - assert!( - stderr.contains("config.missing"), - "expected missing config path in stderr, got:\n{stderr}" - ); -} - -#[test] -fn compile_compose_rejects_whole_config_program_image_ref() { - let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .expect("cli crate should live under the workspace root"); - - let outputs_root = workspace_root.join("target").join("cli-test-outputs"); - fs::create_dir_all(&outputs_root).expect("failed to create outputs directory"); - let outputs_dir = tempfile::Builder::new() - .prefix("program-image-whole-config-check-") - .tempdir_in(&outputs_root) - .expect("failed to create outputs directory"); - - let manifest = outputs_dir.path().join("scenario.json5"); - fs::write( - &manifest, - r##"{ - manifest_version: "0.1.0", - config_schema: { - type: "object", - properties: { - image: { type: "string" } - } - }, - program: { - image: "${config}", - entrypoint: ["run"], + path: "/bin/echo", + args: ["admin", "${slots.ctl.url}"], network: { endpoints: [ { name: "http", port: 8080, protocol: "http" } @@ -1104,76 +538,26 @@ fn compile_compose_rejects_whole_config_program_image_ref() { } "##, ) - .expect("failed to write manifest"); - - let artifact_dir = outputs_dir.path().join("compose"); - let output = Command::new(env!("CARGO_BIN_EXE_amber")) - .arg("compile") - .arg("--docker-compose") - .arg(&artifact_dir) - .arg(&manifest) - .output() - .unwrap_or_else(|err| panic!("failed to run amber compile --docker-compose: {err}")); - - assert!( - !output.status.success(), - "amber compile --docker-compose unexpectedly succeeded" - ); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!( - stderr.contains("program.image cannot") - && stderr.contains("reference the entire runtime config object"), - "expected whole-config image rejection in stderr, got:\n{stderr}" - ); -} - -#[test] -fn compile_vm_rejects_whole_config_vm_image_ref() { - let workspace_root = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .expect("cli crate should live under the workspace root"); - - let outputs_root = workspace_root.join("target").join("cli-test-outputs"); - fs::create_dir_all(&outputs_root).expect("failed to create outputs directory"); - let outputs_dir = tempfile::Builder::new() - .prefix("vm-image-whole-config-check-") - .tempdir_in(&outputs_root) - .expect("failed to create outputs directory"); + .expect("failed to write admin manifest"); let manifest = outputs_dir.path().join("scenario.json5"); fs::write( &manifest, r##"{ - manifest_version: "0.1.0", - config_schema: { - type: "object", - properties: { - image: { type: "string" } - } - }, - program: { - vm: { - image: "${config}", - cpus: 1, - memory_mib: 512, - network: { - endpoints: [ - { name: "http", port: 8080, protocol: "http" } - ], - egress: "none" - } - } - }, - provides: { - http: { kind: "http", endpoint: "http" } + manifest_version: "0.3.0", + components: { + admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], exports: { - http: "http" + admin_http: "#admin.http" } } "##, ) - .expect("failed to write manifest"); + .expect("failed to write root manifest"); let artifact_dir = outputs_dir.path().join("vm"); let output = Command::new(env!("CARGO_BIN_EXE_amber")) @@ -1186,13 +570,16 @@ fn compile_vm_rejects_whole_config_vm_image_ref() { assert!( !output.status.success(), - "amber compile --vm unexpectedly succeeded" + "amber compile --vm unexpectedly succeeded\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) ); let stderr = String::from_utf8_lossy(&output.stderr); assert!( - stderr.contains("program.vm.image cannot") - && stderr.contains("reference the entire runtime config object"), - "expected whole-config vm image rejection in stderr, got:\n{stderr}" + stderr.contains("explicit") + && stderr.contains("controlling direct site") + && stderr.contains("mixed-site placement"), + "expected controller-site rejection in stderr, got:\n{stderr}" ); } diff --git a/cli/tests/compile_outputs/mod.rs b/cli/tests/compile_outputs/mod.rs index 070827e6..581af5d1 100644 --- a/cli/tests/compile_outputs/mod.rs +++ b/cli/tests/compile_outputs/mod.rs @@ -4,10 +4,7 @@ use std::{ process::Command, }; -use amber_compiler::reporter::{ - direct::{DIRECT_CONTROL_SOCKET_RELATIVE_PATH, DIRECT_PLAN_VERSION}, - vm::{VM_PLAN_FILENAME, VM_PLAN_VERSION}, -}; +use amber_compiler::reporter::direct::{DIRECT_CONTROL_SOCKET_RELATIVE_PATH, DIRECT_PLAN_VERSION}; use amber_images::AMBER_ROUTER; use amber_manifest::ManifestDigest; use amber_template::{ diff --git a/cli/tests/mixed_run/dry_run.rs b/cli/tests/mixed_run/dry_run.rs index 673bbc76..f29e68e9 100644 --- a/cli/tests/mixed_run/dry_run.rs +++ b/cli/tests/mixed_run/dry_run.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeSet; + use super::*; #[test] @@ -176,15 +178,66 @@ fn mixed_run_dry_run_emits_launch_bundle_without_starting_sites() { let stitching = launch_bundle["stitching"] .as_array() .expect("launch bundle stitching should serialize as an array"); - assert_eq!(stitching.len(), 1); - assert_eq!(stitching[0]["provider_site"], json!("compose_local")); - assert_eq!(stitching[0]["consumer_site"], json!("direct_local")); + let controller_prefix = "/__amber_internal_framework_component_controller/"; + let controller_stitching = stitching + .iter() + .filter(|link| { + link["provider_component"] + .as_str() + .is_some_and(|component| component.starts_with(controller_prefix)) + || link["consumer_component"] + .as_str() + .is_some_and(|component| component.starts_with(controller_prefix)) + }) + .collect::>(); + let user_stitching = stitching + .iter() + .filter(|link| { + !link["provider_component"] + .as_str() + .is_some_and(|component| component.starts_with(controller_prefix)) + && !link["consumer_component"] + .as_str() + .is_some_and(|component| component.starts_with(controller_prefix)) + }) + .collect::>(); + assert_eq!(user_stitching.len(), 1); + assert_eq!( + controller_stitching.len(), + 2, + "the two-site fixture has program components on both sites, so dynamic capabilities need \ + controller routes in both directions", + ); + let controller_pairs = controller_stitching + .iter() + .map(|link| { + ( + link["provider_site"] + .as_str() + .expect("controller stitching should record provider site") + .to_string(), + link["consumer_site"] + .as_str() + .expect("controller stitching should record consumer site") + .to_string(), + ) + }) + .collect::>(); + assert_eq!( + controller_pairs, + BTreeSet::from([ + ("compose_local".to_string(), "direct_local".to_string()), + ("direct_local".to_string(), "compose_local".to_string()), + ]), + ); + assert_eq!(user_stitching[0]["provider_site"], json!("compose_local")); + assert_eq!(user_stitching[0]["consumer_site"], json!("direct_local")); assert_eq!( - stitching[0]["resolution"], + user_stitching[0]["resolution"], json!("requires_runtime_discovery") ); assert!( - stitching[0]["unresolved_reason"] + user_stitching[0]["unresolved_reason"] .as_str() .is_some_and(|reason| reason.contains("Docker")), "compose-backed stitching should explain why the exact external URL is not known" diff --git a/cli/tests/mixed_run/smoke.rs b/cli/tests/mixed_run/smoke.rs index e2949737..f1aef60e 100644 --- a/cli/tests/mixed_run/smoke.rs +++ b/cli/tests/mixed_run/smoke.rs @@ -812,11 +812,43 @@ fn framework_site_controller_state_path(run: &RunHandle) -> PathBuf { .expect("run should materialize at least one site controller state file") } +fn framework_controller_internal_route_id(run: &RunHandle, site_id: &str) -> String { + let state = read_json(&framework_site_controller_state_path(run)); + let controller_moniker = state["base_scenario"]["components"] + .as_array() + .expect("site controller state should include base scenario components") + .iter() + .find(|component| { + component["metadata"]["kind"] + == amber_compiler::run_plan::FRAMEWORK_COMPONENT_CONTROLLER_METADATA_KIND + && component["metadata"]["execution_site"] == site_id + }) + .and_then(|component| component["moniker"].as_str()) + .expect("base scenario should include the local synthetic site controller component"); + amber_mesh::component_route_id( + controller_moniker, + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + amber_mesh::MeshProtocol::Http, + ) +} + fn framework_controller_post(run: &RunHandle, path: &str, payload: &Value) -> (u16, String) { let plan = read_json(&framework_site_controller_plan_path(run)); let authority_url = plan["authority_url"] .as_str() .expect("site controller plan should publish authority_url"); + let site_id = plan["site_id"] + .as_str() + .expect("site controller plan should publish site_id"); + let route_id = framework_controller_internal_route_id(run, site_id); + let logical_component_id = payload["holder_component_id"] + .as_str() + .or_else(|| payload["caller_component_id"].as_str()) + .expect("dynamic capability controller request should identify the calling component"); + let peer_id = logical_component_id + .strip_prefix("components.") + .filter(|moniker| moniker.starts_with('/')) + .expect("dynamic capability logical component id should be a component moniker"); let body = serde_json::to_string(payload).expect("request body should serialize"); let output = command_output_via_tempfiles( std::process::Command::new("curl") @@ -828,22 +860,11 @@ fn framework_controller_post(run: &RunHandle, path: &str, payload: &Value) -> (u .arg("-H") .arg("content-type: application/json") .arg("-H") - .arg("-H") .arg("x-amber-site-controller-local-only: 1") .arg("-H") - .arg(format!( - "x-amber-route-id: site-controller:{}", - plan["site_id"] - .as_str() - .expect("site controller plan should publish site_id") - )) + .arg(format!("x-amber-route-id: {route_id}")) .arg("-H") - .arg(format!( - "x-amber-peer-id: {}", - plan["router_identity_id"] - .as_str() - .expect("site controller plan should publish router identity") - )) + .arg(format!("x-amber-peer-id: {peer_id}")) .arg("--data") .arg(body) .arg("-o") @@ -1708,7 +1729,10 @@ fn framework_component_direct_create_destroy_live() { } }), ); - assert_eq!(create_status, 200, "create request should succeed"); + assert_eq!( + create_status, 200, + "create request should succeed: {create_response}" + ); let create_json: Value = serde_json::from_str(&create_response).expect("create response should be valid json"); assert_eq!(create_json["child"]["name"], "job-1"); @@ -3179,7 +3203,10 @@ fn framework_component_compose_parent_standby_direct_live() { "/create/worker/job-1", "create request should return an HTTP response", ); - assert_eq!(create_status, 200, "create request should succeed"); + assert_eq!( + create_status, 200, + "create request should succeed: {create_response}" + ); let create_json: Value = serde_json::from_str(&create_response).expect("create response should be valid json"); assert_eq!(create_json["child"]["name"], "job-1"); @@ -3317,7 +3344,10 @@ fn framework_component_direct_parent_compose_child_live() { "/create/job-1", "create request should return an HTTP response", ); - assert_eq!(create_status, 200, "create request should succeed"); + assert_eq!( + create_status, 200, + "create request should succeed: {create_response}" + ); let create_json: Value = serde_json::from_str(&create_response).expect("create response should be valid json"); assert_eq!(create_json["child"]["name"], "job-1"); diff --git a/cli/tests/mixed_run/support.rs b/cli/tests/mixed_run/support.rs index 955109f4..b59ca102 100644 --- a/cli/tests/mixed_run/support.rs +++ b/cli/tests/mixed_run/support.rs @@ -140,6 +140,7 @@ ThreadingHTTPServer(("0.0.0.0", PORT), Handler).serve_forever() const TEST_APP_SOURCE_IMAGE: &str = "python:3.13-alpine"; const TEST_APP_LOCAL_IMAGE_REPOSITORY: &str = "amber-mixed-run-test-app"; +const KUBERNETES_SITE_CONTROLLER_MAIN_CONTAINER: &str = "main"; #[derive(Clone, Debug, PartialEq, Eq)] struct DockerImageMeta { @@ -1433,6 +1434,10 @@ pub(crate) fn framework_control_state_snapshot(control_state_root: &Path) -> Val let Some(site_id) = entry.file_name().into_string().ok() else { continue; }; + let manager_state = site_manager_state_from_state_root(control_state_root, &site_id); + if !site_hosts_local_framework_controller(control_state_root, &site_id, &manager_state) { + continue; + } let state = framework_site_control_state(control_state_root, &site_id); if let Some(children) = state["live_children"].as_array() { live_children.extend(children.iter().cloned()); @@ -1472,6 +1477,20 @@ fn framework_site_control_state(control_state_root: &Path, site_id: &str) -> Val read_json(&state_path) } +fn site_hosts_local_framework_controller( + control_state_root: &Path, + site_id: &str, + manager_state: &Value, +) -> bool { + if manager_state["kind"].as_str() == Some("kubernetes") { + return control_state_root + .join(site_id) + .join("site-controller-plan.json") + .is_file(); + } + framework_site_control_state_path(control_state_root, site_id, manager_state).is_file() +} + fn site_manager_state(run_root: &Path, site_id: &str) -> Value { site_manager_state_from_state_root(&run_root.join("state"), site_id) } @@ -1560,6 +1579,8 @@ fn kubernetes_child_runtime_root_exists(manager_state: &Value, child_id: u64) -> .arg(namespace) .arg("exec") .arg(&pod) + .arg("-c") + .arg(KUBERNETES_SITE_CONTROLLER_MAIN_CONTAINER) .arg("--") .arg("sh") .arg("-lc") @@ -1622,6 +1643,8 @@ fn materialize_kubernetes_child_artifact( .arg("-n") .arg(namespace) .arg("cp") + .arg("-c") + .arg(KUBERNETES_SITE_CONTROLLER_MAIN_CONTAINER) .arg(format!("{pod}:{remote_artifact_dir}")) .arg(&target) .status() @@ -1668,6 +1691,8 @@ fn materialize_kubernetes_control_state(manager_state: &Value, state_path: &Path .arg(namespace) .arg("exec") .arg(&pod) + .arg("-c") + .arg(KUBERNETES_SITE_CONTROLLER_MAIN_CONTAINER) .arg("--") .arg("cat") .arg("/amber/site/state/site-controller-state.json") @@ -3198,6 +3223,47 @@ mod tests { ); } + #[test] + fn framework_control_state_snapshot_skips_sites_without_local_controller() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let state_root = temp.path().join("state"); + let direct_root = state_root.join("direct_local"); + let vm_root = state_root.join("vm_local"); + fs::create_dir_all(&direct_root).expect("direct state root should create"); + fs::create_dir_all(&vm_root).expect("vm state root should create"); + write_json( + &direct_root.join("manager-state.json"), + &json!({ + "kind": "direct", + "site_id": "direct_local", + }), + ); + write_json( + &direct_root.join("site-controller-state.json"), + &json!({ + "live_children": [ + { + "name": "local-child", + "state": "live", + "child_id": 7, + } + ], + }), + ); + write_json( + &vm_root.join("manager-state.json"), + &json!({ + "kind": "vm", + "site_id": "vm_local", + }), + ); + + let snapshot = framework_control_state_snapshot(&state_root); + + assert_eq!(snapshot["live_children"].as_array().unwrap().len(), 1); + assert_eq!(snapshot["live_children"][0]["name"], "local-child"); + } + #[cfg(unix)] #[test] fn kind_load_image_archive_uses_real_archive_path() { diff --git a/compiler/src/run_plan.rs b/compiler/src/run_plan.rs index e3b130c3..9145a8b4 100644 --- a/compiler/src/run_plan.rs +++ b/compiler/src/run_plan.rs @@ -7,7 +7,7 @@ use amber_json5 as json5; use amber_manifest::{ BindingSource as ManifestBindingSource, BindingTarget as ManifestBindingTarget, CapabilityKind, ChildTemplateDecl, ChildTemplateManifestDecl, ComponentDecl, Manifest, ManifestRef, - MountSource, NetworkProtocol, Program as ManifestProgram, SlotDecl, + MountSource, NetworkProtocol, Program as ManifestProgram, RuntimeBackend, SlotDecl, }; use amber_mesh::dynamic_caps::DynamicCapabilitiesSnapshotIr; use amber_scenario::{ @@ -230,6 +230,11 @@ pub(crate) fn lower_framework_component_bindings_for_single_site( scenario, &offered_sites, &assignments_by_component, + &PlacementDefaults { + path: Some(site_id.to_string()), + image: Some(site_id.to_string()), + vm: Some(site_id.to_string()), + }, &controller_sites, )?; compiled @@ -258,6 +263,7 @@ pub fn lower_framework_component_bindings_for_placement( scenario, &offered_sites, &assignments_by_component, + &placement.defaults, &BTreeSet::new(), )?; compiled @@ -411,6 +417,12 @@ pub fn build_run_plan_with_activation( .filter(|site_id| !control_only_site_set.contains(*site_id)) .cloned() .collect::>(); + let privileged_control_sites = initial_privileged_control_sites( + compiled.scenario(), + &offered_sites, + &assignments_by_component, + &standby_sites, + )?; let active_site_capabilities: BTreeMap = initial_active_sites .iter() @@ -420,7 +432,7 @@ pub fn build_run_plan_with_activation( ActiveSiteCapabilities { cross_site_routing: true, dynamic_workloads: dynamic_enabled_sites.contains(site_id), - privileged_control: true, + privileged_control: privileged_control_sites.contains(site_id), }, ) }) @@ -434,21 +446,13 @@ pub fn build_run_plan_with_activation( ) }; - let controller_sites = if has_public_framework_component_bindings(compiled.scenario()) { - active_site_capabilities - .keys() - .map(|site_id| framework_controller_execution_site_id(&offered_sites, site_id)) - .collect::, _>>()? - .into_iter() - .map(str::to_string) - .collect::>() - } else { - BTreeSet::new() - }; + let controller_sites = + controller_execution_sites_for_active_sites(&offered_sites, &active_site_capabilities)?; let lowered = inject_site_controller_components( compiled.scenario(), &offered_sites, &assignments_by_component, + &defaults, &controller_sites, )?; assignments_by_component.extend(lowered.synthetic_assignments.clone()); @@ -655,6 +659,13 @@ pub fn build_homogeneous_export_run_plan( compiled: &CompiledScenario, requested_kind: SiteKind, ) -> Result { + if requested_kind == SiteKind::Vm { + return Err(RunPlanError::Other( + "homogeneous vm exports are unsupported; vm workloads require an explicit controlling \ + direct site via placement.sites..controller_site" + .to_string(), + )); + } let site_id = homogeneous_export_site_id(requested_kind).to_string(); let placement = PlacementFile { schema: PLACEMENT_SCHEMA.to_string(), @@ -868,22 +879,15 @@ fn resolve_dynamic_planning_state( explicit_components, )?; validate_storage_locality(scenario, &assignments_by_component)?; - let controller_sites = if has_public_framework_component_bindings(scenario) { - activation - .active_site_capabilities - .keys() - .map(|site_id| framework_controller_execution_site_id(&offered_sites, site_id)) - .collect::, _>>()? - .into_iter() - .map(str::to_string) - .collect::>() - } else { - BTreeSet::new() - }; + let controller_sites = controller_execution_sites_for_active_sites( + &offered_sites, + &activation.active_site_capabilities, + )?; let lowered = inject_site_controller_components( scenario, &offered_sites, &assignments_by_component, + &placement.defaults, &controller_sites, )?; assignments_by_component.extend(lowered.synthetic_assignments.clone()); @@ -1239,6 +1243,60 @@ fn framework_controller_execution_site_id<'a>( } } +fn controller_execution_sites_for_active_sites( + site_definitions: &BTreeMap, + active_site_capabilities: &BTreeMap, +) -> Result, RunPlanError> { + active_site_capabilities + .iter() + .filter(|(_, capabilities)| capabilities.privileged_control) + .map(|(site_id, _)| framework_controller_execution_site_id(site_definitions, site_id)) + .collect::, _>>() + .map(|sites| sites.into_iter().map(str::to_string).collect()) +} + +fn initial_privileged_control_sites( + scenario: &Scenario, + _offered_sites: &BTreeMap, + assignments_by_component: &BTreeMap, + standby_sites: &[String], +) -> Result, RunPlanError> { + let moniker_to_site = assignments_by_component + .iter() + .map(|(component_id, site_id)| { + ( + graph::component_path(scenario, *component_id), + site_id.clone(), + ) + }) + .collect::>(); + let mut privileged_control_sites = standby_sites.iter().cloned().collect::>(); + privileged_control_sites.extend(assignments_by_component.values().cloned()); + for (component_id, component) in scenario.components_iter() { + if component.child_templates.is_empty() { + continue; + } + if let Some(site_id) = assignments_by_component.get(&component_id) { + privileged_control_sites.insert(site_id.clone()); + } + } + for binding in &scenario.bindings { + let BindingFrom::Framework(framework) = &binding.from else { + continue; + }; + if framework.capability.as_str() != "component" { + continue; + } + privileged_control_sites.insert(framework_component_managed_site_id( + scenario, + &moniker_to_site, + framework.authority, + binding.to.component, + )?); + } + Ok(privileged_control_sites) +} + fn expand_sites_to_execution_sites( site_ids: &BTreeSet, site_definitions: &BTreeMap, @@ -1254,6 +1312,7 @@ fn expand_sites_to_execution_sites( #[derive(Clone, Debug)] struct FrozenChildTemplateSpec { manifests: Option>, + possible_backends: Vec, } #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] @@ -1283,16 +1342,20 @@ fn analyze_standby_sites( let mut requested_sites = BTreeSet::new(); for template in collect_frozen_child_templates(scenario)? { - let Some(keys) = template.manifests.as_ref() else { - continue; - }; - for key in keys { - requested_sites.extend(analyze_manifest_root_sites( - &scenario.manifest_catalog, + if let Some(keys) = template.manifests.as_ref() { + for key in keys { + requested_sites.extend(analyze_manifest_root_sites( + &scenario.manifest_catalog, + site_definitions, + defaults, + key, + )?); + } + } else { + requested_sites.extend(site_ids_for_possible_backends( site_definitions, - defaults, - key, - )?); + &template.possible_backends, + )); } } @@ -1308,6 +1371,7 @@ fn collect_frozen_child_templates( templates.extend(component.child_templates.values().map(|template| { FrozenChildTemplateSpec { manifests: template.manifests.clone(), + possible_backends: template.possible_backends.clone(), } })); } @@ -1351,7 +1415,10 @@ fn freeze_manifest_child_template( } }; - Ok(FrozenChildTemplateSpec { manifests }) + Ok(FrozenChildTemplateSpec { + manifests, + possible_backends: template.possible_backends.clone(), + }) } fn resolve_catalog_key( @@ -1408,6 +1475,80 @@ fn analyze_manifest_root_sites( Ok(analyzer.required_sites) } +fn framework_component_authority_child_execution_sites( + scenario: &Scenario, + site_definitions: &BTreeMap, + defaults: &PlacementDefaults, + authority_realm: ComponentId, +) -> Result, RunPlanError> { + let mut sites = BTreeSet::new(); + for template in scenario.component(authority_realm).child_templates.values() { + sites.extend(child_template_execution_sites( + scenario, + site_definitions, + defaults, + template, + )?); + } + Ok(sites) +} + +fn child_template_execution_sites( + scenario: &Scenario, + site_definitions: &BTreeMap, + defaults: &PlacementDefaults, + template: &amber_scenario::ChildTemplate, +) -> Result, RunPlanError> { + let mut requested_sites = BTreeSet::new(); + if let Some(keys) = template.manifests.as_ref() { + for key in keys { + requested_sites.extend(analyze_manifest_root_sites( + &scenario.manifest_catalog, + site_definitions, + defaults, + key, + )?); + } + } else { + requested_sites.extend(site_ids_for_possible_backends( + site_definitions, + &template.possible_backends, + )); + } + requested_sites + .iter() + .map(|site_id| framework_controller_execution_site_id(site_definitions, site_id)) + .collect::, _>>() + .map(|sites| sites.into_iter().map(str::to_string).collect()) +} + +fn site_ids_for_possible_backends( + site_definitions: &BTreeMap, + possible_backends: &[RuntimeBackend], +) -> BTreeSet { + if possible_backends.is_empty() { + return BTreeSet::new(); + } + let possible_backends = possible_backends.iter().copied().collect::>(); + site_definitions + .iter() + .filter_map(|(site_id, site)| { + possible_backends + .contains(&runtime_backend_for_site_kind(site.kind)) + .then_some(site_id.clone()) + }) + .collect() +} + +fn runtime_backend_for_site_kind(kind: SiteKind) -> RuntimeBackend { + match kind { + SiteKind::Direct => RuntimeBackend::Direct, + SiteKind::Vm => RuntimeBackend::Vm, + SiteKind::Compose => RuntimeBackend::Compose, + SiteKind::Kubernetes => RuntimeBackend::Kubernetes, + } +} + impl FragmentAnalyzer<'_> { fn walk( &mut self, @@ -2173,6 +2314,8 @@ const FRAMEWORK_COMPONENT_CONTROLLER_ENDPOINT_NAME: &str = "framework_component" const FRAMEWORK_COMPONENT_CONTROLLER_MONIKER_PREFIX: &str = "/__amber_internal_framework_component_controller"; const FRAMEWORK_COMPONENT_CONTROLLER_PROVIDE_PREFIX: &str = "__amber_internal_framework_component"; +const FRAMEWORK_COMPONENT_CONTROLLER_REMOTE_SLOT_PREFIX: &str = + "__amber_internal_framework_component_controller_remote"; const FRAMEWORK_COMPONENT_CONTROLLER_PORT_BASE: u16 = 32000; const FRAMEWORK_COMPONENT_CONTROLLER_CONTAINER_PATH: &str = "/usr/local/bin/amber-site-controller"; const FRAMEWORK_COMPONENT_CONTROLLER_PLAN_PATH: &str = @@ -2191,6 +2334,7 @@ fn inject_site_controller_components( scenario: &Scenario, offered_sites: &BTreeMap, assignments_by_component: &BTreeMap, + defaults: &PlacementDefaults, controller_sites: &BTreeSet, ) -> Result { let framework_bindings = scenario @@ -2203,7 +2347,7 @@ fn inject_site_controller_components( _ => None, }) .collect::>(); - if framework_bindings.is_empty() { + if framework_bindings.is_empty() && controller_sites.is_empty() { return Ok(LoweredFrameworkComponentScenario { scenario: scenario.clone(), synthetic_assignments: BTreeMap::new(), @@ -2221,6 +2365,32 @@ fn inject_site_controller_components( .collect::>(); let mut grants_by_execution_site = BTreeMap::>::new(); + let mut remote_controller_edges = BTreeSet::<(String, String)>::new(); + let dynamic_capability_execution_sites = scenario + .components_iter() + .filter(|(_, component)| component.program.is_some()) + .map(|(component_id, _)| { + assignments_by_component + .get(&component_id) + .ok_or_else(|| { + RunPlanError::Other(format!( + "program component `{}` has no site assignment", + graph::component_path(scenario, component_id) + )) + }) + .and_then(|site_id| { + framework_controller_execution_site_id(offered_sites, site_id) + .map(str::to_string) + }) + }) + .collect::, _>>()?; + for local_site_id in &dynamic_capability_execution_sites { + for remote_site_id in &dynamic_capability_execution_sites { + if local_site_id != remote_site_id { + remote_controller_edges.insert((local_site_id.clone(), remote_site_id.clone())); + } + } + } for (binding, framework) in &framework_bindings { let managed_site = framework_component_managed_site_id( scenario, @@ -2230,6 +2400,32 @@ fn inject_site_controller_components( )?; let execution_site = framework_controller_execution_site_id(offered_sites, &managed_site)?.to_string(); + let recipient_moniker = graph::component_path(scenario, binding.to.component); + let recipient_site = moniker_to_site + .get(&recipient_moniker) + .ok_or_else(|| { + RunPlanError::Other(format!( + "framework.component recipient `{recipient_moniker}` has no site assignment" + )) + })? + .clone(); + let recipient_execution_site = + framework_controller_execution_site_id(offered_sites, &recipient_site)?.to_string(); + if recipient_execution_site != execution_site { + remote_controller_edges + .insert((recipient_execution_site.clone(), execution_site.clone())); + } + for target_execution_site in framework_component_authority_child_execution_sites( + scenario, + offered_sites, + defaults, + framework.authority, + )? { + if target_execution_site != recipient_execution_site { + remote_controller_edges + .insert((recipient_execution_site.clone(), target_execution_site)); + } + } grants_by_execution_site .entry(execution_site) .or_default() @@ -2254,6 +2450,49 @@ fn inject_site_controller_components( let mut rewritten = scenario.clone(); let mut synthetic_assignments = BTreeMap::new(); let mut controller_rewrites = BTreeMap::::new(); + for (component_id, component) in rewritten.components_iter() { + let Some(metadata) = framework_component_controller_metadata(component.metadata.as_ref()) + else { + continue; + }; + if !offered_sites.contains_key(&metadata.execution_site) { + return Err(RunPlanError::UnknownSite { + site_id: metadata.execution_site, + }); + } + let provide_by_grant = metadata + .grants + .into_iter() + .map(|(provide, grant)| { + ( + FrameworkComponentGrantKey { + managed_site: grant.managed_site, + authority_realm: ComponentId(grant.authority_realm_id), + authority_realm_moniker: grant.authority_realm_moniker, + }, + provide, + ) + }) + .collect(); + synthetic_assignments + .entry(component_id) + .or_insert_with(|| metadata.execution_site.clone()); + if controller_rewrites + .insert( + metadata.execution_site.clone(), + ControllerComponentRewrite { + component_id, + provide_by_grant, + }, + ) + .is_some() + { + return Err(RunPlanError::Other(format!( + "site `{}` already has multiple synthetic framework.component controllers", + metadata.execution_site + ))); + } + } let execution_sites = controller_sites .iter() @@ -2264,6 +2503,20 @@ fn inject_site_controller_components( let grants = grants_by_execution_site .remove(&execution_site) .unwrap_or_default(); + if let Some(rewrite) = controller_rewrites.get(&execution_site) { + let missing_grants = grants + .iter() + .filter(|grant| !rewrite.provide_by_grant.contains_key(*grant)) + .collect::>(); + if !missing_grants.is_empty() { + return Err(RunPlanError::Other(format!( + "site `{execution_site}` already has a synthetic framework.component \ + controller, but it is missing {} lowered framework.component grant(s)", + missing_grants.len() + ))); + } + continue; + } let controller_id = ComponentId(rewritten.components.len()); let port = next_framework_component_controller_port(&mut used_ports)?; let site = offered_sites @@ -2331,6 +2584,44 @@ fn inject_site_controller_components( ); } + for (local_site_id, remote_site_id) in remote_controller_edges { + let local_controller_id = controller_rewrites + .get(&local_site_id) + .ok_or_else(|| { + RunPlanError::Other(format!( + "site `{local_site_id}` needs a remote-controller route to `{remote_site_id}` \ + but has no synthetic framework.component controller" + )) + })? + .component_id; + let remote_controller_id = controller_rewrites + .get(&remote_site_id) + .ok_or_else(|| { + RunPlanError::Other(format!( + "site `{local_site_id}` needs a remote-controller route to `{remote_site_id}` \ + but the remote site has no synthetic framework.component controller" + )) + })? + .component_id; + let slot_name = framework_component_controller_remote_slot_name(&remote_site_id); + rewritten.component_mut(local_controller_id).slots.insert( + slot_name.clone(), + framework_component_controller_remote_slot_decl()?, + ); + rewritten.bindings.push(BindingEdge { + from: BindingFrom::Component(ProvideRef { + component: remote_controller_id, + name: FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE.to_string(), + }), + to: SlotRef { + component: local_controller_id, + name: slot_name, + }, + // Remote controller routes must exist without introducing a cross-site startup cycle. + weak: true, + }); + } + for binding in &mut rewritten.bindings { let BindingFrom::Framework(framework) = &binding.from else { continue; @@ -2428,6 +2719,13 @@ fn framework_component_controller_provide_name(grant: &FrameworkComponentGrantKe ) } +pub fn framework_component_controller_remote_slot_name(site_id: &str) -> String { + format!( + "{FRAMEWORK_COMPONENT_CONTROLLER_REMOTE_SLOT_PREFIX}__{}", + sanitize_framework_component_identifier(site_id), + ) +} + fn unique_framework_component_controller_moniker( scenario: &Scenario, execution_site: &str, @@ -2556,6 +2854,14 @@ fn framework_component_controller_internal_provide_decl() }) } +fn framework_component_controller_remote_slot_decl() -> Result { + serde_json::from_value(serde_json::json!({ "kind": "http" })).map_err(|err| { + RunPlanError::Other(format!( + "failed to build synthetic remote site controller slot: {err}" + )) + }) +} + #[cfg(test)] mod tests { use std::{ @@ -2897,6 +3203,28 @@ mod tests { CompiledScenario::from_compile_output(&output).expect("compiled scenario") } + fn direct_mesh_config_for_component( + site_plan: &RunSitePlan, + moniker: &str, + ) -> amber_mesh::MeshConfigTemplate { + let mesh_provision_plan: amber_mesh::MeshProvisionPlan = serde_json::from_str( + site_plan + .artifact_files + .get(crate::targets::direct::MESH_PROVISION_PLAN_FILENAME) + .expect("mesh provision plan should be present in site artifacts"), + ) + .expect("mesh provision plan should deserialize"); + mesh_provision_plan + .targets + .into_iter() + .find_map(|target| { + (matches!(target.kind, amber_mesh::MeshProvisionTargetKind::Component) + && target.config.identity.id == moniker) + .then_some(target.config) + }) + .unwrap_or_else(|| panic!("mesh provision plan should include component {moniker}")) + } + #[tokio::test] async fn vm_site_defaults_controller_site_to_sole_direct_site() { let dir = tmp_dir("run-plan-vm-controller-default-"); @@ -3102,7 +3430,7 @@ mod tests { assert!(plan.sites.contains_key("guest_vm")); assert_eq!( plan.sites["host_direct"].assigned_components, - Vec::::new() + vec!["/__amber_internal_framework_component_controller/host_direct".to_string()] ); assert_eq!( plan.sites["guest_vm"].assigned_components, @@ -3137,7 +3465,45 @@ mod tests { } #[tokio::test] - async fn sites_without_public_framework_bindings_do_not_inject_site_controller_components() { + async fn compile_only_lowering_without_framework_bindings_does_not_inject_site_controller_components() + { + let dir = tmp_dir("compile-framework-lowering-no-controller-"); + let worker = dir.path().join("worker.json5"); + let root = dir.path().join("root.json5"); + + write(&worker, path_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { worker: "./worker.json5" }, + exports: { worker_api: "#worker.api" } +}"##, + ); + + let compiled = compile(&root).await; + let lowered = lower_framework_component_bindings_for_single_site( + &compiled, + "direct_local", + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ) + .expect("compile-time framework lowering should succeed"); + let scenario = lowered.scenario(); + + assert!( + scenario.components_iter().all(|(_, component)| { + framework_component_controller_metadata(component.metadata.as_ref()).is_none() + }), + "compile-time lowering should stay inert when no framework.component bindings exist" + ); + } + + #[tokio::test] + async fn build_run_plan_injects_site_controller_components_for_component_hosting_sites() { let dir = tmp_dir("run-plan-controller-injection-"); let worker = dir.path().join("worker.json5"); let root = dir.path().join("root.json5"); @@ -3160,12 +3526,81 @@ mod tests { .expect("direct site should be present"); let scenario = Scenario::try_from(site_plan.scenario_ir.clone()) .expect("site scenario should round-trip from IR"); + let worker_config = direct_mesh_config_for_component(site_plan, "/worker"); + framework_controller_component_for_site(&scenario, "direct_local"); assert!( - scenario.components_iter().all(|(_, component)| { - framework_component_controller_metadata(component.metadata.as_ref()).is_none() + plan.active_site_capabilities["direct_local"].privileged_control, + "component-hosting sites should advertise privileged control because dynamic \ + capability control is site-local", + ); + assert!( + worker_config.dynamic_caps_listen.is_some(), + "components on populated sites should expose dynamic caps listeners once a local \ + controller exists", + ); + assert!( + worker_config.outbound.iter().any(|route| { + route.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME }), - "sites without public framework.component bindings should not inject a controller" + "components on populated sites should get the synthetic local controller route", + ); + } + + #[tokio::test] + async fn build_run_plan_direct_artifacts_keep_local_controller_routes_for_framework_sites() { + let dir = tmp_dir("run-plan-direct-artifact-controller-route-"); + let admin = dir.path().join("admin.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_path_manifest()); + write( + &root, + r###"{ + manifest_version: "0.3.0", + components: { admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + exports: { admin_api: "#admin.api" } +}"###, + ); + + let compiled = compile(&root).await; + let run_plan = build_run_plan(&compiled, None).expect("run plan should build"); + let site_plan = run_plan + .sites + .get("direct_local") + .expect("direct site should be present"); + let site_scenario = Scenario::try_from(site_plan.scenario_ir.clone()) + .expect("site scenario should round-trip from IR"); + let (_, controller_component, _) = + framework_controller_component_for_site(&site_scenario, "direct_local"); + let admin_config = direct_mesh_config_for_component(site_plan, "/admin"); + let controller_config = + direct_mesh_config_for_component(site_plan, controller_component.moniker.as_str()); + + assert!( + run_plan.active_site_capabilities["direct_local"].privileged_control, + "sites that provide framework.component should advertise privileged control", + ); + assert!( + admin_config.dynamic_caps_listen.is_some(), + "ordinary direct-site components should expose dynamic caps listeners when a local \ + framework controller exists", + ); + assert!( + admin_config.outbound.iter().any(|route| { + route.protocol == amber_mesh::MeshProtocol::Http + && route.capability + == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + }), + "framework-enabled direct-site components must keep their local framework.component \ + controller route", + ); + assert!( + controller_config.dynamic_caps_listen.is_none(), + "synthetic controller sidecars must not expose the dynamic caps listener", ); } @@ -3221,6 +3656,49 @@ mod tests { assert!(plan.links.is_empty()); } + #[tokio::test] + async fn framework_component_lowering_reuses_existing_synthetic_controller_on_replay() { + let dir = tmp_dir("run-plan-framework-component-replay-"); + let admin = dir.path().join("admin.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_path_manifest()); + write( + &root, + r###"{ + manifest_version: "0.3.0", + components: { admin: "./admin.json5" }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + exports: { admin_api: "#admin.api" } +}"###, + ); + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, None).expect("run plan should build"); + let replay_compiled = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered scenario should round-trip through scenario IR"); + let replay_plan = + build_run_plan(&replay_compiled, None).expect("replay run plan should build"); + let replay = CompiledScenario::from_ir(replay_plan.base_scenario) + .expect("replay scenario should deserialize"); + let controllers = replay + .scenario() + .components_iter() + .filter(|(_, component)| { + framework_component_controller_metadata(component.metadata.as_ref()).is_some() + }) + .collect::>(); + + assert_eq!( + controllers.len(), + 1, + "replaying a snapshot must reuse the existing synthetic controller instead of \ + injecting a second controller for the same site", + ); + } + #[tokio::test] async fn framework_component_lowering_routes_cross_site_authorities_to_their_managed_site_controller() { @@ -3250,6 +3728,8 @@ mod tests { let scenario = lowered.scenario(); let (controller_id, controller, metadata) = framework_controller_component_for_site(scenario, "compose_local"); + let (direct_controller_id, direct_controller, _) = + framework_controller_component_for_site(scenario, "direct_local"); let binding = scenario .bindings .iter() @@ -3276,6 +3756,321 @@ mod tests { assert_eq!(ctl_link.provider_site, "compose_local"); assert_eq!(ctl_link.consumer_site, "direct_local"); assert_eq!(ctl_link.provider_component, controller.moniker.as_str()); + let remote_controller_binding = scenario + .bindings + .iter() + .find(|binding| { + binding.to.component == direct_controller_id + && binding.to.name + == framework_component_controller_remote_slot_name("compose_local") + }) + .expect( + "synthetic direct-site controller should bind to the remote compose controller \ + internal route", + ); + let BindingFrom::Component(remote_controller_provide) = &remote_controller_binding.from + else { + panic!("remote controller binding should lower to an ordinary component provide"); + }; + assert_eq!(remote_controller_provide.component, controller_id); + assert_eq!( + remote_controller_provide.name, + FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE, + ); + assert!( + remote_controller_binding.weak, + "remote controller bindings should not impose cross-site startup ordering", + ); + assert_eq!( + graph::component_path(scenario, remote_controller_binding.to.component), + direct_controller.moniker.as_str(), + ); + assert!( + scenario.bindings.iter().any(|binding| { + binding.to.component == controller_id + && binding.to.name + == framework_component_controller_remote_slot_name("direct_local") + }), + "component-hosting sites keep reverse controller routes because dynamic capabilities \ + can be shared in either direction at runtime", + ); + } + + #[tokio::test] + async fn framework_component_lowering_routes_creators_to_standby_child_site_controllers() { + let dir = tmp_dir("run-plan-framework-component-standby-controller-"); + let admin = dir.path().join("admin.json5"); + let worker = dir.path().join("worker.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_path_manifest()); + write(&worker, image_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + slots: { realm: { kind: "component", optional: true } }, + components: { admin: "./admin.json5" }, + child_templates: { + worker: { manifest: "./worker.json5" } + }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + exports: { admin_api: "#admin.api" } +}"##, + ); + let placement = PlacementFile { + schema: PLACEMENT_SCHEMA.to_string(), + version: PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "compose_local".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + controller_site: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_local".to_string()), + image: Some("compose_local".to_string()), + vm: None, + }, + components: BTreeMap::new(), + dynamic_capabilities: None, + framework_children: None, + }; + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, Some(&placement)).expect("run plan should build"); + assert_eq!(plan.standby_sites, vec!["compose_local".to_string()]); + let lowered = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered scenario should deserialize"); + let scenario = lowered.scenario(); + let (direct_controller_id, direct_controller, _) = + framework_controller_component_for_site(scenario, "direct_local"); + let (compose_controller_id, _compose_controller, compose_metadata) = + framework_controller_component_for_site(scenario, "compose_local"); + assert!( + compose_metadata.grants.is_empty(), + "the compose controller is needed as a standby create target, not because it owns an \ + existing framework.component grant", + ); + + let remote_controller_binding = scenario + .bindings + .iter() + .find(|binding| { + binding.to.component == direct_controller_id + && binding.to.name + == framework_component_controller_remote_slot_name("compose_local") + }) + .expect("direct controller should have a route to the standby compose controller"); + let BindingFrom::Component(remote_controller_provide) = &remote_controller_binding.from + else { + panic!("remote controller binding should lower to an ordinary component provide"); + }; + assert_eq!(remote_controller_provide.component, compose_controller_id); + assert_eq!( + remote_controller_provide.name, + FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE, + ); + assert_eq!( + graph::component_path(scenario, remote_controller_binding.to.component), + direct_controller.moniker.as_str(), + ); + assert!(remote_controller_binding.weak); + assert!( + scenario.bindings.iter().all(|binding| { + binding.to.component != compose_controller_id + || binding.to.name + != framework_component_controller_remote_slot_name("direct_local") + }), + "creating into the compose standby site should not synthesize an unused reverse route", + ); + } + + #[tokio::test] + async fn framework_component_lowering_routes_vm_template_creators_to_controller_site() { + let dir = tmp_dir("run-plan-framework-component-vm-template-controller-"); + let admin = dir.path().join("admin.json5"); + let worker = dir.path().join("worker.json5"); + let root = dir.path().join("root.json5"); + + write( + &admin, + r#"{ + manifest_version: "0.3.0", + slots: { ctl: { kind: "component" } }, + program: { + image: "busybox:1.36.1", + entrypoint: ["sh", "-c", "sleep 30"], + env: { CTL_URL: "${slots.ctl.url}" }, + network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] } + }, + provides: { api: { kind: "http", endpoint: "http" } }, + exports: { api: "api" } +}"#, + ); + write(&worker, vm_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + slots: { realm: { kind: "component", optional: true } }, + components: { admin: "./admin.json5" }, + child_templates: { + worker: { manifest: "./worker.json5" } + }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + exports: { admin_api: "#admin.api" } +}"##, + ); + let placement = PlacementFile { + schema: PLACEMENT_SCHEMA.to_string(), + version: PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "compose_local".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + controller_site: None, + }, + ), + ( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "vm_local".to_string(), + SiteDefinition { + kind: SiteKind::Vm, + context: None, + controller_site: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_local".to_string()), + image: Some("compose_local".to_string()), + vm: Some("vm_local".to_string()), + }, + components: BTreeMap::new(), + dynamic_capabilities: None, + framework_children: None, + }; + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, Some(&placement)).expect("run plan should build"); + assert_eq!( + plan.standby_sites, + vec!["direct_local".to_string(), "vm_local".to_string()] + ); + let lowered = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered scenario should deserialize"); + let scenario = lowered.scenario(); + let (compose_controller_id, _compose_controller, _) = + framework_controller_component_for_site(scenario, "compose_local"); + let (direct_controller_id, _direct_controller, direct_metadata) = + framework_controller_component_for_site(scenario, "direct_local"); + assert!( + direct_metadata.grants.is_empty(), + "the direct controller is needed because it controls vm creates, not because it owns \ + an existing framework.component grant", + ); + + let remote_controller_binding = scenario + .bindings + .iter() + .find(|binding| { + binding.to.component == compose_controller_id + && binding.to.name + == framework_component_controller_remote_slot_name("direct_local") + }) + .expect("compose controller should route vm creates to the direct controller"); + let BindingFrom::Component(remote_controller_provide) = &remote_controller_binding.from + else { + panic!("remote controller binding should lower to an ordinary component provide"); + }; + assert_eq!(remote_controller_provide.component, direct_controller_id); + assert_eq!( + remote_controller_provide.name, + FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE, + ); + assert!( + scenario.bindings.iter().all(|binding| { + binding.to.component != compose_controller_id + || binding.to.name + != framework_component_controller_remote_slot_name("vm_local") + }), + "vm workload sites do not host framework.component controllers; the route target is \ + the vm controller_site", + ); + } + + #[tokio::test] + async fn framework_component_lowering_does_not_synthesize_legacy_controller_peer_bindings() { + let dir = tmp_dir("run-plan-framework-component-legacy-controller-peer-"); + let admin = dir.path().join("admin.json5"); + let authority = dir.path().join("authority.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_path_manifest()); + write(&authority, framework_authority_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { authority: "./authority.json5" }, + exports: { + authority_api: "#authority.api", + authority_admin_api: "#authority.admin_api" + } +}"##, + ); + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, None).expect("run plan should build"); + let lowered = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered base scenario should deserialize"); + let scenario = lowered.scenario(); + + assert!( + scenario.components_iter().all(|(_, component)| { + component.slots.keys().all(|slot| { + !slot.starts_with("__amber_internal_framework_component_controller_peer") + }) + }), + "framework controller lowering must not synthesize legacy controller-peer slots", + ); + assert!( + scenario.bindings.iter().all(|binding| { + !matches!( + &binding.from, + BindingFrom::Component(ProvideRef { name, .. }) + if name == "__amber_internal_site_controller_peer" + ) + }), + "framework controller lowering must not synthesize legacy controller-peer capability \ + bindings", + ); } #[tokio::test] @@ -3476,11 +4271,14 @@ mod tests { assert_eq!(plan.defaults.path.as_deref(), Some("direct_local")); assert_eq!( plan.sites["compose_local"].assigned_components, - vec!["/".to_string()] + vec![ + "/".to_string(), + "/__amber_internal_framework_component_controller/compose_local".to_string(), + ] ); assert_eq!( plan.sites["direct_local"].assigned_components, - Vec::::new() + vec!["/__amber_internal_framework_component_controller/direct_local".to_string()] ); assert!( plan.sites["direct_local"] @@ -3695,7 +4493,16 @@ mod tests { "direct_local".to_string() ] ); - assert_eq!(plan.links.len(), 6); + let controller_prefix = "/__amber_internal_framework_component_controller/"; + let user_link_count = plan + .links + .iter() + .filter(|link| { + !link.provider_component.starts_with(controller_prefix) + && !link.consumer_component.starts_with(controller_prefix) + }) + .count(); + assert_eq!(user_link_count, 6); } #[test] @@ -3988,4 +4795,35 @@ mod tests { .contains_key(&PathBuf::from("kustomization.yaml")) ); } + + #[tokio::test] + async fn build_homogeneous_export_run_plan_rejects_vm_without_controller_site() { + let dir = tmp_dir("run-plan-homogeneous-vm-export-"); + let vm_child = dir.path().join("vm.json5"); + let root = dir.path().join("root.json5"); + + write(&vm_child, vm_server_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + components: { + vm: "./vm.json5" + }, + exports: { + vm_api: "#vm.api" + } +}"##, + ); + + let compiled = compile(&root).await; + let err = build_homogeneous_export_run_plan(&compiled, SiteKind::Vm) + .expect_err("homogeneous vm export should require an explicit controlling site"); + let rendered = err.to_string(); + assert!( + rendered.contains("homogeneous vm exports are unsupported"), + "{rendered}" + ); + assert!(rendered.contains("controller_site"), "{rendered}"); + } } diff --git a/compiler/src/targets/direct/mod.rs b/compiler/src/targets/direct/mod.rs index 19511233..da7bde62 100644 --- a/compiler/src/targets/direct/mod.rs +++ b/compiler/src/targets/direct/mod.rs @@ -489,17 +489,6 @@ fn build_component_plans( component.moniker.as_str() )) })?; - let dynamic_caps_port = mesh_config_plan - .component_configs - .get(id) - .and_then(|config| config.dynamic_caps_listen) - .map(|addr| addr.port()) - .ok_or_else(|| { - MeshError::new(format!( - "internal error: missing dynamic caps listen port for {}", - component.moniker.as_str() - )) - })?; let depends_on = mesh_plan .strong_deps() .get(id) @@ -515,6 +504,14 @@ fn build_component_plans( component.moniker.as_str(), )? }; + let dynamic_caps_port = match execution { + DirectProgramExecutionPlan::InternalSiteController => None, + _ => mesh_config_plan + .component_configs + .get(id) + .and_then(|config| config.dynamic_caps_listen) + .map(|addr| addr.port()), + }; out.push(DirectComponentPlan { id: id.0, @@ -553,17 +550,19 @@ fn direct_internal_execution_plan( fn inject_direct_dynamic_caps_env( execution: DirectProgramExecutionPlan, - dynamic_caps_port: u16, + dynamic_caps_port: Option, ) -> DirectProgramExecutionPlan { match execution { DirectProgramExecutionPlan::Direct { entrypoint, mut env, } => { - env.insert( - DYNAMIC_CAPS_API_URL_ENV.to_string(), - format!("http://127.0.0.1:{dynamic_caps_port}"), - ); + if let Some(dynamic_caps_port) = dynamic_caps_port { + env.insert( + DYNAMIC_CAPS_API_URL_ENV.to_string(), + format!("http://127.0.0.1:{dynamic_caps_port}"), + ); + } DirectProgramExecutionPlan::Direct { entrypoint, env } } DirectProgramExecutionPlan::HelperRunner { diff --git a/compiler/src/targets/mesh/docker_compose/mod.rs b/compiler/src/targets/mesh/docker_compose/mod.rs index b9558156..5aa1290c 100644 --- a/compiler/src/targets/mesh/docker_compose/mod.rs +++ b/compiler/src/targets/mesh/docker_compose/mod.rs @@ -113,7 +113,7 @@ fn build_control_socket_init_service(volume_name: &str) -> Service { format!( "mkdir -p {ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER} && chown \ {ROUTER_RUNTIME_UID}:{ROUTER_RUNTIME_GID} {ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER} \ - && chmod 0700 {ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER}" + && chmod 0770 {ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER}" ), ]); service.volumes.push(format!( diff --git a/compiler/src/targets/mesh/docker_compose/tests/rendering.rs b/compiler/src/targets/mesh/docker_compose/tests/rendering.rs index cc068d4f..dd8d3dc8 100644 --- a/compiler/src/targets/mesh/docker_compose/tests/rendering.rs +++ b/compiler/src/targets/mesh/docker_compose/tests/rendering.rs @@ -1301,6 +1301,13 @@ fn compose_emits_export_metadata_and_labels() { ); let control_init_service = service(&compose, "amber-router-control-init"); assert_eq!(control_init_service.user.as_deref(), Some("0:0")); + assert!( + control_init_service + .command + .as_ref() + .is_some_and(|command| command.iter().any(|arg| arg.contains("chmod 0770"))), + "router control dir should be group-accessible to the injected site controller" + ); assert!( control_init_service .volumes diff --git a/compiler/src/targets/mesh/kubernetes/mod.rs b/compiler/src/targets/mesh/kubernetes/mod.rs index 55d5b36a..e0ae365b 100644 --- a/compiler/src/targets/mesh/kubernetes/mod.rs +++ b/compiler/src/targets/mesh/kubernetes/mod.rs @@ -1281,8 +1281,13 @@ pub(crate) fn emit_kubernetes_artifact_with_options( } let egress_from_consumers = egress_allow.get(id); - let egress_to_router = egress_router_allow.get(id); - if egress_from_consumers.is_some() || egress_to_router.is_some() { + let mut egress_to_router = egress_router_allow.get(id).cloned().unwrap_or_default(); + if framework_component_controller_metadata(s.component(*id).metadata.as_ref()).is_some() + && needs_router + { + egress_to_router.insert(ROUTER_CONTROL_PORT_BASE); + } + if egress_from_consumers.is_some() || !egress_to_router.is_empty() { netpol.add_egress_rule(NetworkPolicyEgressRule { to: vec![NetworkPolicyPeer { pod_selector: None, @@ -1344,7 +1349,7 @@ pub(crate) fn emit_kubernetes_artifact_with_options( } } - if let Some(ports) = egress_to_router { + if !egress_to_router.is_empty() { netpol.add_egress_rule(NetworkPolicyEgressRule { to: vec![NetworkPolicyPeer { pod_selector: Some(LabelSelector { @@ -1353,7 +1358,7 @@ pub(crate) fn emit_kubernetes_artifact_with_options( namespace_selector: None, ip_block: None, }], - ports: ports + ports: egress_to_router .iter() .map(|port| NetworkPolicyPort { protocol: "TCP", diff --git a/compiler/src/targets/mesh/kubernetes/tests/mod.rs b/compiler/src/targets/mesh/kubernetes/tests/mod.rs index 99f57d16..9165219e 100644 --- a/compiler/src/targets/mesh/kubernetes/tests/mod.rs +++ b/compiler/src/targets/mesh/kubernetes/tests/mod.rs @@ -82,9 +82,11 @@ fn lowered_framework_component_controller_uses_real_internal_image_in_kubernetes r##" { manifest_version: "0.1.0", + slots: { external: { kind: "http" } }, components: { admin: "./admin.json5" }, bindings: [ - { to: "#admin.ctl", from: "framework.component" } + { to: "#admin.ctl", from: "framework.component" }, + { to: "#admin.external", from: "self.external", weak: true } ] } "##, @@ -99,7 +101,10 @@ fn lowered_framework_component_controller_uses_real_internal_image_in_kubernetes image: "busybox:1.36.1", entrypoint: ["sh", "-lc", "sleep 3600"] }, - slots: { ctl: { kind: "component" } } + slots: { + ctl: { kind: "component" }, + external: { kind: "http" } + } } "#, ) @@ -115,6 +120,12 @@ fn lowered_framework_component_controller_uses_real_internal_image_in_kubernetes .files .get(&PathBuf::from("04-services/amber-site-controller.yaml")) .expect("site controller service"); + let netpol = artifact + .files + .get(&PathBuf::from( + "05-networkpolicies/amber-site-controller-netpol.yaml", + )) + .expect("site controller network policy"); let site_controller_image = internal_images().site_controller; assert!( @@ -131,6 +142,30 @@ fn lowered_framework_component_controller_uses_real_internal_image_in_kubernetes "the lowered controller should render an ordinary service for the controller workload: \ {service}" ); + let netpol_doc: serde_yaml::Value = + serde_yaml::from_str(netpol).expect("parse site controller network policy"); + let egress = netpol_doc["spec"]["egress"] + .as_sequence() + .expect("site controller network policy should have egress rules"); + let has_router_control_egress = egress.iter().any(|rule| { + let to_router = rule["to"].as_sequence().is_some_and(|peers| { + peers.iter().any(|peer| { + peer["podSelector"]["matchLabels"]["amber.io/component"].as_str() + == Some("amber-router") + }) + }); + let has_control_port = rule["ports"].as_sequence().is_some_and(|ports| { + ports.iter().any(|port| { + port["protocol"].as_str() == Some("TCP") && port["port"].as_u64() == Some(24100) + }) + }); + to_router && has_control_port + }); + assert!( + has_router_control_egress, + "site controller must be able to reach the local router control port without opening a \ + broader egress surface: {netpol}" + ); } fn parse_rendered_env(content: &str) -> std::collections::BTreeMap { diff --git a/compiler/src/targets/mesh/kubernetes/tests/rendering.rs b/compiler/src/targets/mesh/kubernetes/tests/rendering.rs index 29f40b1b..73d33469 100644 --- a/compiler/src/targets/mesh/kubernetes/tests/rendering.rs +++ b/compiler/src/targets/mesh/kubernetes/tests/rendering.rs @@ -712,7 +712,7 @@ fn kubernetes_emits_otelcol_and_wires_otel_env() { } #[test] -fn kubernetes_templates_dynamic_caps_sidecar_control_env() { +fn kubernetes_templates_omit_dynamic_caps_sidecar_control_env_without_local_controller() { let dir = tempdir().expect("temp dir"); let root_path = dir.path().join("root.json5"); let worker_path = dir.path().join("worker.json5"); @@ -763,57 +763,43 @@ fn kubernetes_templates_dynamic_caps_sidecar_control_env() { }) .expect("worker deployment"); assert!( - component_deploy.contains("AMBER_DYNAMIC_CAPS_API_URL"), + !component_deploy.contains("AMBER_DYNAMIC_CAPS_API_URL"), "{component_deploy}" ); assert!( - component_deploy.contains("http://127.0.0.1:19000"), + !component_deploy.contains("http://127.0.0.1:19000"), "{component_deploy}" ); assert!( - component_deploy.contains("amber-component-sidecar-env"), + !component_deploy.contains("amber-component-sidecar-env"), "{component_deploy}" ); - let sidecar_env = artifact + if let Some(sidecar_env) = artifact .files .get(&PathBuf::from(super::COMPONENT_SIDECAR_ENV_FILE)) - .expect("component sidecar env template"); - assert!( - !sidecar_env.contains("AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL="), - "controller URLs must not be injected as ambient sidecar env: {sidecar_env}" - ); - assert!( - !sidecar_env.contains("AMBER_FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN="), - "controller auth must not be injected as ambient sidecar env: {sidecar_env}" - ); - assert!( - sidecar_env.contains("AMBER_DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64="), - "{sidecar_env}" - ); + { + assert!( + !sidecar_env.contains("AMBER_FRAMEWORK_COMPONENT_CONTROLLER_URL="), + "controller URLs must not be injected as ambient sidecar env: {sidecar_env}" + ); + assert!( + !sidecar_env.contains("AMBER_FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN="), + "controller auth must not be injected as ambient sidecar env: {sidecar_env}" + ); + assert!( + !sidecar_env.contains("AMBER_DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64="), + "dynamic caps verify keys must not be injected when no local controller exists: \ + {sidecar_env}" + ); + } let kustomization = artifact .files .get(&PathBuf::from("kustomization.yaml")) .expect("kustomization"); assert!( - kustomization.contains("name: amber-component-sidecar-env"), - "{kustomization}" - ); - assert!( - kustomization.contains(super::COMPONENT_SIDECAR_ENV_FILE), + !kustomization.contains("amber-component-sidecar-env"), "{kustomization}" ); - assert!( - !kustomization.contains(&format!( - "resources:\n- {}", - super::COMPONENT_SIDECAR_ENV_FILE - )), - "{kustomization}" - ); - assert!( - !kustomization.contains(&format!("\n- {}\n", super::COMPONENT_SIDECAR_ENV_FILE)), - "kustomization resources must not treat the sidecar env template as a manifest: \ - {kustomization}" - ); } diff --git a/compiler/src/targets/mesh/mesh_config.rs b/compiler/src/targets/mesh/mesh_config.rs index 49206ed9..429bf35b 100644 --- a/compiler/src/targets/mesh/mesh_config.rs +++ b/compiler/src/targets/mesh/mesh_config.rs @@ -200,11 +200,12 @@ pub(crate) fn build_mesh_config_plan( let mut component_configs: HashMap = HashMap::new(); let mut component_sidecar_env_passthrough = Vec::new(); push_env_passthrough_once(&mut component_sidecar_env_passthrough, SCENARIO_RUN_ID_ENV); - - push_env_passthrough_once( - &mut component_sidecar_env_passthrough, - DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, - ); + if local_controller_component.is_some() { + push_env_passthrough_once( + &mut component_sidecar_env_passthrough, + DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, + ); + } for &id in mesh_plan.program_components() { let identity = identities_by_component .get(&id) @@ -446,11 +447,16 @@ pub(crate) fn build_mesh_config_plan( let mesh_listen = format!("{}:{mesh_port}", options.component_mesh_listen_addr) .parse() .expect("mesh listen"); - let dynamic_caps_listen = route_ports.dynamic_caps_port(id).map(|port| { - format!("{}:{port}", options.component_mesh_listen_addr) - .parse() - .expect("dynamic caps listen") - }); + let dynamic_caps_listen = + if local_controller_component.is_some() && Some(id) != local_controller_component { + route_ports.dynamic_caps_port(id).map(|port| { + format!("{}:{port}", options.component_mesh_listen_addr) + .parse() + .expect("dynamic caps listen") + }) + } else { + None + }; let config_peers = required_peers(&identity.id, &inbound, &outbound); let config = MeshConfigTemplate { @@ -938,6 +944,19 @@ mod tests { vec!["/consumer".to_string(), "/site/test/router".to_string()], "controller internal routing must admit local component sidecars and the site router", ); + assert!( + controller_config.dynamic_caps_listen.is_none(), + "synthetic controller sidecars should not expose the ordinary dynamic caps listener", + ); + + let consumer_config = plan + .component_configs + .get(&ComponentId(0)) + .expect("consumer config"); + assert!( + consumer_config.dynamic_caps_listen.is_some(), + "ordinary components should still expose the dynamic caps listener", + ); } #[test] diff --git a/runtime/mesh/src/lib.rs b/runtime/mesh/src/lib.rs index a9a818b140c4da61a16377cc31a4b5f329d39f86..99dbfe72b2dd6cf3564a41ec39f9abdc2f9c3b9b 100644 GIT binary patch delta 1014 zcmZ`&F>ljA7$v9+UJwHUkyiR*3fOAXNJuQ#HUm^b&?c#X4I#_5&xuw0&bm9Nsi>77 z00WXIBqW%Sm@3qj-@wMo%*sFDJKIUzih4SK-@W&}_wMe~=f>A>jSu$shkx%(0|rvL z2N87?jT1+5?~p1-aIP#H+FjUp)#z~=z*__GyZhP;hE~eLiT$Q1lx5wjHWCsD9$RKd z#ffPbdorVjWk_7$OM(X#v1X9UCxo&wA%zq_6xNI&%UjLUt5?kYOY>&_r@1y;t}Wl6 zEyLxXw=QvVwzKlSeXU~OH1gA@x6gh*f3%c;>|bw~d24Wdqo%NQm;dUur$fdCl@3}# zF%C+@OcsuDnhHJG8$8)1rFePFJVFM{qkEX+R$-{D^-@JATbAAJ!uJ3662j<_Wp5Tb zbQlp;Fm;BhD||%g8S|EIZw&F2xNt%iD(mE=PFaj9`7GNm6Vwp>ZA7T6O0}$DjZPBk zDa%BEp!*<4JdJ!{Q?B5ELc&AI8THKz^0)|xA#)?tSDf0|6%Ln{`Kq_S^Xx@^cgmz2 z&{}kQc7c&-195ZEI+z{&?*u&Z>8g+&FO`bN?L=^oO4-&oqt2We`Mv&+={%d*Ro)2Z z$`*&EC%)t{wZ?9gQX96rFjIAzl4@tRK4;e?eT?EFdX*4CV?L(myx35KqR`fEYbn9! zl)vk*6xWU(+UL2=@&bc)3FT7+jgSyy41jM3Fq~sJ3e^Y_DrT`j896r5n=}`s)rnFZO14LLbEmg63m=NO#^gKxl}xo<04mfZnE(I) diff --git a/runtime/router/src/control.rs b/runtime/router/src/control.rs index c98ff7fc..d68c667d 100644 --- a/runtime/router/src/control.rs +++ b/runtime/router/src/control.rs @@ -758,7 +758,7 @@ pub(super) fn bind_unix_listener(path: &str) -> Result Option { fn resolve_dynamic_caps_controller_env( config: &MeshConfig, ) -> Result, RouterError> { - let local_control_url = local_dynamic_caps_controller_url(config); - let verify_key_raw = nonempty_env_var(amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV); - if local_control_url.is_none() && verify_key_raw.is_none() { + if config.dynamic_caps_listen.is_none() { return Ok(None); } + + let Some(verify_key_raw) = nonempty_env_var(amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV) + else { + return Ok(None); + }; + + let local_control_url = local_dynamic_caps_controller_url(config); let run_id = required_dynamic_caps_env_var(SCENARIO_RUN_ID_ENV, nonempty_env_var(SCENARIO_RUN_ID_ENV))?; let control_url = local_control_url.ok_or_else(|| { @@ -1143,10 +1148,7 @@ fn resolve_dynamic_caps_controller_env( })?; Ok(Some(DynamicCapsControllerEnv { control_url, - verify_key_raw: required_dynamic_caps_env_var( - amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, - verify_key_raw, - )?, + verify_key_raw, run_id, })) } @@ -1229,6 +1231,12 @@ mod tests { config } + fn mesh_config_without_dynamic_caps_listener() -> MeshConfig { + let mut config = mesh_config_with_internal_controller_route(); + config.dynamic_caps_listen = None; + config + } + #[test] fn resolve_dynamic_caps_controller_env_disables_listener_when_control_env_is_absent() { let _guard = ENV_LOCK @@ -1248,6 +1256,28 @@ mod tests { ); } + #[test] + fn resolve_dynamic_caps_controller_env_ignores_control_env_when_listener_is_disabled() { + let _guard = ENV_LOCK + .get_or_init(|| Mutex::new(())) + .lock() + .expect("env lock"); + let _env = EnvGuard::replace([ + ( + amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, + Some("verify-key"), + ), + (SCENARIO_RUN_ID_ENV, Some("run-1234")), + ]); + + assert!( + resolve_dynamic_caps_controller_env(&mesh_config_without_dynamic_caps_listener()) + .expect("dynamic caps env should resolve") + .is_none(), + "sidecars without a dynamic caps listener must ignore controller env", + ); + } + #[test] fn resolve_dynamic_caps_controller_env_rejects_missing_local_controller_route() { let _guard = ENV_LOCK diff --git a/runtime/site-controller/src/control_state_api.rs b/runtime/site-controller/src/control_state_api.rs index 5c469f5c..8992e38c 100644 --- a/runtime/site-controller/src/control_state_api.rs +++ b/runtime/site-controller/src/control_state_api.rs @@ -3,8 +3,8 @@ use base64::Engine as _; use super::{ dynamic_caps, orchestration::{ - ProtocolApiError, load_site_manager_state, local_router_identity_for_overlay, - publish_dynamic_capability_origin, + ProtocolApiError, load_site_manager_state, publish_dynamic_capability_origin, + router_identity_for_overlay, }, planner::{ControlStateApp, live_assignment_map, protocol_error}, state::{FrameworkControlState, persist_control_state_update}, @@ -227,7 +227,7 @@ pub(crate) async fn resolve_dynamic_capability_origin_internal( )) })?; let _origin_runtime = local_component_runtime(app, &state, &root.holder_component_id)?; - let origin_peer = local_router_identity_for_overlay(app).await?; + let origin_peer = router_identity_for_overlay(app, &app.controller_plan.site_id).await?; let origin_state = load_site_manager_state(app, &app.controller_plan.site_id)?; let origin_receipt = super::orchestration::site_receipt_from_manager_state(&origin_state); let origin_peer_addr = if let Some(router_mesh_addr) = diff --git a/runtime/site-controller/src/default_runtime.rs b/runtime/site-controller/src/default_runtime.rs index ad1a5ea8..913ef75e 100644 --- a/runtime/site-controller/src/default_runtime.rs +++ b/runtime/site-controller/src/default_runtime.rs @@ -48,7 +48,9 @@ mod kubernetes_controller; mod site_artifacts; mod site_runtime_support; -pub(crate) use self::child_runtime::default_site_controller_runtime; +pub(crate) use self::child_runtime::{ + default_site_controller_runtime, runtime_plan_for_site_from_controller_plan, +}; pub(super) use self::site_artifacts::*; pub use self::{ child_runtime::cleanup_dynamic_site_children, @@ -1153,7 +1155,7 @@ async fn wait_for_kubernetes_site_router_ready( kubernetes_router_ready_targets(plan, manager_state.as_ref())? && probe_kubernetes_router_control_ready(&control_target, Duration::from_millis(250)) .await? - && router_mesh_listener_ready_target(&mesh_target).await + && router_mesh_listener_ready_target(&mesh_target, Duration::from_millis(250)).await { return Ok(()); } @@ -1223,8 +1225,11 @@ fn kubernetes_local_router_mesh_target(plan: &SiteControllerRuntimePlan) -> Resu Ok(format!("{host}:{mesh_port}")) } -async fn router_mesh_listener_ready_target(target: &str) -> bool { - tokio::net::TcpStream::connect(target).await.is_ok() +async fn router_mesh_listener_ready_target(target: &str, timeout: Duration) -> bool { + matches!( + tokio::time::timeout(timeout, tokio::net::TcpStream::connect(target)).await, + Ok(Ok(_)) + ) } async fn probe_kubernetes_router_control_ready(target: &str, timeout: Duration) -> Result { diff --git a/runtime/site-controller/src/default_runtime/child_runtime.rs b/runtime/site-controller/src/default_runtime/child_runtime.rs index 66aa9a11..70b92246 100644 --- a/runtime/site-controller/src/default_runtime/child_runtime.rs +++ b/runtime/site-controller/src/default_runtime/child_runtime.rs @@ -9,6 +9,8 @@ pub(super) struct SiteControllerRuntimeState { pub(super) kind: SiteKind, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub(super) children: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub(super) direct_input_overlay_providers: BTreeSet, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -17,12 +19,12 @@ pub(super) struct SiteControllerRuntimeChildRecord { pub(super) artifact_root: String, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub(super) assigned_components: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(super) controller_routes: Vec, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub(super) proxy_exports: BTreeMap, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub(super) direct_inputs: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub(super) routed_inputs: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] pub(super) process_pid: Option, pub(super) published: bool, @@ -89,9 +91,10 @@ impl SiteControllerRuntime for DefaultSiteControllerRuntime { plan: &'a SiteControllerPlan, state: FrameworkControlState, child: LiveChildRecord, + site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async move { - let app = self.runtime_app(plan)?; + let app = self.runtime_app_for_site(plan, site_id)?; site_controller_runtime_prepare_child(&app, &state, &child).await }) } @@ -101,9 +104,10 @@ impl SiteControllerRuntime for DefaultSiteControllerRuntime { plan: &'a SiteControllerPlan, state: FrameworkControlState, child: LiveChildRecord, + site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async move { - let app = self.runtime_app(plan)?; + let app = self.runtime_app_for_site(plan, site_id)?; site_controller_runtime_publish_child(&app, &state, &child).await }) } @@ -112,9 +116,10 @@ impl SiteControllerRuntime for DefaultSiteControllerRuntime { &'a self, plan: &'a SiteControllerPlan, child_id: u64, + site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async move { - let app = self.runtime_app(plan)?; + let app = self.runtime_app_for_site(plan, site_id)?; site_controller_runtime_rollback_child(&app, child_id).await }) } @@ -124,9 +129,10 @@ impl SiteControllerRuntime for DefaultSiteControllerRuntime { plan: &'a SiteControllerPlan, state: FrameworkControlState, child: LiveChildRecord, + site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async move { - let app = self.runtime_app(plan)?; + let app = self.runtime_app_for_site(plan, site_id)?; site_controller_runtime_destroy_child(&app, &state, &child).await }) } @@ -200,10 +206,13 @@ impl SiteControllerRuntime for DefaultSiteControllerRuntime { } impl DefaultSiteControllerRuntime { - fn runtime_app(&self, plan: &SiteControllerPlan) -> Result { - self.runtime_app_for_plan( - &crate::runtime_api::site_controller_runtime_plan_from_controller_plan(plan), - ) + fn runtime_app_for_site( + &self, + controller_plan: &SiteControllerPlan, + site_id: &str, + ) -> Result { + let runtime_plan = runtime_plan_for_site_from_controller_plan(controller_plan, site_id)?; + self.runtime_app_for_plan(&runtime_plan) } fn runtime_app_for_plan( @@ -238,6 +247,7 @@ pub(super) fn build_site_controller_runtime_app( site_id: plan.site_id.clone(), kind: plan.kind, children: BTreeMap::new(), + direct_input_overlay_providers: BTreeSet::new(), }; write_json(&state_path, &state)?; state @@ -249,6 +259,90 @@ pub(super) fn build_site_controller_runtime_app( }) } +pub(crate) fn runtime_plan_for_site_from_controller_plan( + controller_plan: &SiteControllerPlan, + site_id: &str, +) -> Result { + if site_id == controller_plan.site_id { + return Ok(site_controller_runtime_plan_from_controller_plan( + controller_plan, + )); + } + + let site_state_root = Path::new(&controller_plan.state_root).join(site_id); + let supervisor: SiteSupervisorPlan = read_json( + &site_supervisor_plan_path(&site_state_root), + "site supervisor plan", + )?; + let manager_state_path = + crate::runtime_api::site_state_path(Path::new(&controller_plan.state_root), site_id); + let manager = manager_state_path + .is_file() + .then(|| read_json::(&manager_state_path, "site manager state")) + .transpose()?; + let local_router_control = manager + .as_ref() + .and_then(|state| state.router_control.clone()) + .or_else(|| { + Some(site_runtime_local_router_control( + supervisor.kind, + Path::new(&supervisor.artifact_dir), + )) + }); + let router_identity_id = manager + .as_ref() + .and_then(|state| state.router_identity_id.clone()) + .or_else(|| { + controller_plan + .peer_router_identities + .get(site_id) + .map(|identity| identity.id.clone()) + }) + .ok_or_else(|| miette::miette!("site `{site_id}` is missing its router identity"))?; + let router_mesh_port = manager + .as_ref() + .and_then(|state| state.router_mesh_addr.as_deref()) + .and_then(|addr| addr.rsplit_once(':')) + .and_then(|(_, port)| port.parse::().ok()) + .or(supervisor.router_mesh_port) + .or(supervisor.port_forward_mesh_port); + + let mut runtime_plan = site_controller_runtime_plan_from_controller_plan(controller_plan); + runtime_plan.run_id = supervisor.run_id; + runtime_plan.mesh_scope = supervisor.mesh_scope; + runtime_plan.run_root = supervisor.run_root; + runtime_plan.site_id = supervisor.site_id; + runtime_plan.kind = supervisor.kind; + runtime_plan.router_identity_id = router_identity_id; + runtime_plan.local_router_control = local_router_control; + runtime_plan.artifact_dir = supervisor.artifact_dir; + runtime_plan.site_state_root = supervisor.site_state_root; + runtime_plan.storage_root = supervisor.storage_root; + runtime_plan.runtime_root = supervisor.runtime_root; + runtime_plan.router_mesh_port = router_mesh_port; + runtime_plan.compose_project = supervisor.compose_project; + runtime_plan.kubernetes_namespace = supervisor.kubernetes_namespace; + runtime_plan.context = supervisor.context; + runtime_plan.observability_endpoint = supervisor.observability_endpoint; + runtime_plan.launch_env = supervisor.launch_env; + Ok(runtime_plan) +} + +fn site_runtime_local_router_control(kind: SiteKind, artifact_dir: &Path) -> String { + match kind { + SiteKind::Direct => format!( + "unix://{}", + super::direct_current_control_socket_path(artifact_dir).display() + ), + SiteKind::Vm => format!( + "unix://{}", + super::vm_current_control_socket_path(artifact_dir).display() + ), + SiteKind::Compose => "unix:///amber/control/router-control.sock".to_string(), + SiteKind::Kubernetes => "amber-router:24100".to_string(), + } +} + #[derive(Clone)] pub(super) struct SiteControllerRuntimeApp { pub(super) plan: SiteControllerRuntimePlan, @@ -397,10 +491,11 @@ pub fn cleanup_dynamic_site_children(site_state_root: &Path, kind: SiteKind) -> ); remove_dynamic_child_root(kind, &child_root, Some(Path::new(&child.artifact_root)))?; } - if state.children.is_empty() { + if state.children.is_empty() && state.direct_input_overlay_providers.is_empty() { return Ok(()); } state.children.clear(); + state.direct_input_overlay_providers.clear(); write_json(&state_path, &state) } @@ -614,23 +709,6 @@ pub(super) async fn site_controller_runtime_prepare_child( &runtime_spec.direct_inputs, &live_components, )?; - rewrite_dynamic_routed_inputs_in_artifact( - &artifact_root, - &runtime_spec.routed_inputs, - app.plan.kind, - &app.plan.router_identity_id, - app.plan.router_mesh_port, - )?; - write_direct_vm_startup_route_overlay_payload( - &artifact_root, - "direct", - &runtime_spec.routed_inputs, - &overlay_peer_addr_map_from_ports(&local_direct_peer_ports_for_children( - &app.plan, - &published_children, - )?), - &local_direct_peer_identities_for_children(&app.plan, &published_children)?, - )?; } SiteKind::Vm => { filter_vm_stage_plan(&artifact_root, &runtime_spec.component_ids)?; @@ -644,23 +722,6 @@ pub(super) async fn site_controller_runtime_prepare_child( &runtime_spec.direct_inputs, &live_components, )?; - rewrite_dynamic_routed_inputs_in_artifact( - &artifact_root, - &runtime_spec.routed_inputs, - app.plan.kind, - &app.plan.router_identity_id, - app.plan.router_mesh_port, - )?; - write_direct_vm_startup_route_overlay_payload( - &artifact_root, - "vm", - &runtime_spec.routed_inputs, - &overlay_peer_addr_map_from_ports(&local_vm_peer_ports_for_children( - &app.plan, - &published_children, - )?), - &local_vm_peer_identities_for_children(&app.plan, &published_children)?, - )?; } SiteKind::Compose => { let existing_peer_identities = @@ -700,9 +761,9 @@ pub(super) async fn site_controller_runtime_prepare_child( child_id: child.child_id, artifact_root: artifact_root.display().to_string(), assigned_components: runtime_spec.assigned_components.clone(), + controller_routes: runtime_spec.controller_routes.clone(), proxy_exports: runtime_spec.proxy_exports.clone(), direct_inputs: runtime_spec.direct_inputs.clone(), - routed_inputs: runtime_spec.routed_inputs.clone(), process_pid: None, published: false, }, @@ -816,7 +877,6 @@ pub(super) async fn site_controller_runtime_publish_child( Path::new(&child.artifact_root), &child.assigned_components, &child.proxy_exports, - &child.routed_inputs, &live_peer_ports, &live_peer_identities, )?; @@ -834,6 +894,7 @@ pub(super) async fn site_controller_runtime_publish_child( drop(state); project_dynamic_direct_router_surface(&app.plan, &child)?; reconcile_dynamic_site_router_overlays(app).await?; + reconcile_dynamic_site_controller_overlay(app).await?; reconcile_dynamic_direct_input_overlays(app).await?; reconcile_site_proxy_metadata( Path::new(&app.plan.artifact_dir), @@ -930,7 +991,6 @@ pub(super) async fn site_controller_runtime_publish_child( Path::new(&child.artifact_root), &child.assigned_components, &child.proxy_exports, - &child.routed_inputs, &live_peer_ports, &live_peer_identities, )?; @@ -948,6 +1008,7 @@ pub(super) async fn site_controller_runtime_publish_child( drop(state); project_dynamic_vm_router_surface(&app.plan, &child)?; reconcile_dynamic_site_router_overlays(app).await?; + reconcile_dynamic_site_controller_overlay(app).await?; reconcile_dynamic_direct_input_overlays(app).await?; reconcile_site_proxy_metadata( Path::new(&app.plan.artifact_dir), @@ -1018,6 +1079,7 @@ pub(super) async fn site_controller_runtime_publish_child( write_json(&app.state_path, &*state)?; } reconcile_dynamic_site_router_overlays(app).await?; + reconcile_dynamic_site_controller_overlay(app).await?; reconcile_dynamic_direct_input_overlays(app).await?; if !workload_services.is_empty() { let status = @@ -1077,7 +1139,6 @@ pub(super) async fn site_controller_runtime_publish_child( artifact_root, &child.assigned_components, &child.proxy_exports, - &child.routed_inputs, &live_peer_identities, )?; apply_dynamic_site_router_overlay(&app.plan, &child).await?; @@ -1090,6 +1151,7 @@ pub(super) async fn site_controller_runtime_publish_child( write_json(&app.state_path, &*state)?; drop(state); reconcile_dynamic_site_router_overlays(app).await?; + reconcile_dynamic_site_controller_overlay(app).await?; reconcile_dynamic_direct_input_overlays(app).await?; wait_for_kubernetes_site_router_ready( &app.plan, @@ -1252,6 +1314,7 @@ pub(super) async fn site_controller_runtime_destroy_child( SiteKind::Direct | SiteKind::Vm | SiteKind::Compose | SiteKind::Kubernetes ) { reconcile_dynamic_site_router_overlays(app).await?; + reconcile_dynamic_site_controller_overlay(app).await?; reconcile_dynamic_direct_input_overlays(app).await?; } let child_root = site_controller_runtime_child_root(&app.plan, child_id); @@ -1288,4 +1351,268 @@ mod tests { ], ); } + + #[test] + fn runtime_plan_for_kubernetes_site_uses_port_forward_mesh_runtime_metadata() { + let temp = tempfile::tempdir().expect("temp dir"); + let state_root = temp.path().join("state"); + let compose_state_root = state_root.join("compose_local"); + let kind_state_root = state_root.join("kind_local"); + let compose_artifact = temp.path().join("artifact").join("compose_local"); + let kind_artifact = temp.path().join("artifact").join("kind_local"); + fs::create_dir_all(&compose_state_root).expect("compose state root"); + fs::create_dir_all(&kind_state_root).expect("kind state root"); + fs::create_dir_all(&compose_artifact).expect("compose artifact"); + fs::create_dir_all(&kind_artifact).expect("kind artifact"); + + write_json( + &site_supervisor_plan_path(&kind_state_root), + &serde_json::json!({ + "schema": "amber.run.site_supervisor_plan", + "version": 2, + "run_id": "test-run", + "mesh_scope": "test-mesh", + "run_root": temp.path().join("run").display().to_string(), + "coordinator_pid": 1u32, + "site_id": "kind_local", + "kind": "kubernetes", + "artifact_dir": kind_artifact.display().to_string(), + "site_state_root": kind_state_root.display().to_string(), + "kubernetes_namespace": "amber-test-kind", + "context": "kind-amber-test", + "port_forward_mesh_port": 24036u16, + "port_forward_control_port": 24037u16, + "launch_env": {"KUBECONFIG": "/tmp/kubeconfig"} + }), + ) + .expect("kubernetes supervisor plan should write"); + write_json( + &crate::runtime_api::site_state_path(&state_root, "kind_local"), + &serde_json::json!({ + "schema": "amber.run.site_manager_state", + "version": 1, + "run_id": "test-run", + "site_id": "kind_local", + "kind": "kubernetes", + "status": "running", + "artifact_dir": kind_artifact.display().to_string(), + "supervisor_pid": 1u32, + "kubernetes_namespace": "amber-test-kind", + "context": "kind-amber-test", + "router_control": "127.0.0.1:24037", + "router_mesh_addr": "127.0.0.1:24036", + "router_identity_id": "/site/kind_local/router", + "router_public_key_b64": base64::engine::general_purpose::STANDARD.encode([8u8; 32]) + }), + ) + .expect("kubernetes manager state should write"); + + let controller_plan = SiteControllerPlan { + schema: "amber.framework_component.site_controller_plan".to_string(), + version: 1, + run_id: "test-run".to_string(), + mesh_scope: "test-mesh".to_string(), + site_id: "compose_local".to_string(), + kind: SiteKind::Compose, + listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), + authority_url: "http://127.0.0.1:0".to_string(), + router_identity_id: "/site/compose_local/router".to_string(), + peer_site_router_urls: BTreeMap::new(), + peer_router_identities: BTreeMap::from([( + "kind_local".to_string(), + MeshIdentityPublic { + id: "/site/kind_local/router".to_string(), + public_key: [8u8; 32], + mesh_scope: Some("test-mesh".to_string()), + }, + )]), + peer_router_mesh_addrs: BTreeMap::new(), + local_router_control: Some("unix:///tmp/compose-control.sock".to_string()), + published_router_mesh_addr: Some("127.0.0.1:24034".to_string()), + compose_consumer_router_mesh_addr: None, + kubernetes_consumer_router_mesh_addr: None, + state_path: compose_state_root + .join("site-controller-state.json") + .display() + .to_string(), + run_root: temp.path().join("run").display().to_string(), + state_root: state_root.display().to_string(), + site_state_root: compose_state_root.display().to_string(), + artifact_dir: compose_artifact.display().to_string(), + control_state_auth_token: "test-auth".to_string(), + dynamic_caps_token_verify_key_b64: String::new(), + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24034), + compose_project: Some("amber-test-compose".to_string()), + kubernetes_namespace: None, + context: None, + observability_endpoint: None, + launch_env: BTreeMap::new(), + }; + + let runtime_plan = + runtime_plan_for_site_from_controller_plan(&controller_plan, "kind_local") + .expect("kubernetes runtime plan should derive from the managed site"); + assert_eq!(runtime_plan.site_id, "kind_local"); + assert_eq!(runtime_plan.kind, SiteKind::Kubernetes); + assert_eq!( + runtime_plan.artifact_dir, + kind_artifact.display().to_string() + ); + assert_eq!( + runtime_plan.site_state_root, + kind_state_root.display().to_string() + ); + assert_eq!( + runtime_plan.local_router_control.as_deref(), + Some("127.0.0.1:24037") + ); + assert_eq!(runtime_plan.router_identity_id, "/site/kind_local/router"); + assert_eq!(runtime_plan.router_mesh_port, Some(24036)); + assert_eq!( + runtime_plan.kubernetes_namespace.as_deref(), + Some("amber-test-kind") + ); + assert_eq!(runtime_plan.context.as_deref(), Some("kind-amber-test")); + assert_eq!( + runtime_plan + .launch_env + .get("KUBECONFIG") + .map(String::as_str), + Some("/tmp/kubeconfig") + ); + } + + #[test] + fn runtime_plan_for_vm_site_uses_managed_vm_runtime_metadata() { + let temp = tempfile::tempdir().expect("temp dir"); + let state_root = temp.path().join("state"); + let direct_state_root = state_root.join("direct_local"); + let vm_state_root = state_root.join("vm_local"); + let direct_artifact = temp.path().join("artifact").join("direct_local"); + let vm_artifact = temp.path().join("artifact").join("vm_local"); + fs::create_dir_all(&direct_state_root).expect("direct state root"); + fs::create_dir_all(&vm_state_root).expect("vm state root"); + fs::create_dir_all(&direct_artifact).expect("direct artifact"); + fs::create_dir_all(&vm_artifact).expect("vm artifact"); + + write_json( + &site_supervisor_plan_path(&vm_state_root), + &serde_json::json!({ + "schema": "amber.run.site_supervisor_plan", + "version": 2, + "run_id": "test-run", + "mesh_scope": "test-mesh", + "run_root": temp.path().join("run").display().to_string(), + "coordinator_pid": 1u32, + "site_id": "vm_local", + "kind": "vm", + "artifact_dir": vm_artifact.display().to_string(), + "site_state_root": vm_state_root.display().to_string(), + "storage_root": temp.path().join("storage").join("vm_local").display().to_string(), + "runtime_root": temp.path().join("runtime").join("vm_local").display().to_string(), + "router_mesh_port": 24001u16, + "launch_env": {"AMBER_TEST": "1"} + }), + ) + .expect("vm supervisor plan should write"); + write_json( + &crate::runtime_api::site_state_path(&state_root, "vm_local"), + &serde_json::json!({ + "schema": "amber.run.site_manager_state", + "version": 1, + "run_id": "test-run", + "site_id": "vm_local", + "kind": "vm", + "status": "running", + "artifact_dir": vm_artifact.display().to_string(), + "supervisor_pid": 1u32, + "router_control": "unix:///tmp/vm-control.sock", + "router_mesh_addr": "127.0.0.1:24001", + "router_identity_id": "/site/vm_local/router", + "router_public_key_b64": base64::engine::general_purpose::STANDARD.encode([9u8; 32]) + }), + ) + .expect("vm manager state should write"); + + let controller_plan = SiteControllerPlan { + schema: "amber.framework_component.site_controller_plan".to_string(), + version: 1, + run_id: "test-run".to_string(), + mesh_scope: "test-mesh".to_string(), + site_id: "direct_local".to_string(), + kind: SiteKind::Direct, + listen_addr: SocketAddr::from(([127, 0, 0, 1], 0)), + authority_url: "http://127.0.0.1:0".to_string(), + router_identity_id: "/site/direct_local/router".to_string(), + peer_site_router_urls: BTreeMap::new(), + peer_router_identities: BTreeMap::from([( + "vm_local".to_string(), + MeshIdentityPublic { + id: "/site/vm_local/router".to_string(), + public_key: [9u8; 32], + mesh_scope: Some("test-mesh".to_string()), + }, + )]), + peer_router_mesh_addrs: BTreeMap::new(), + local_router_control: Some("unix:///tmp/direct-control.sock".to_string()), + published_router_mesh_addr: Some("127.0.0.1:24000".to_string()), + compose_consumer_router_mesh_addr: None, + kubernetes_consumer_router_mesh_addr: None, + state_path: direct_state_root + .join("site-controller-state.json") + .display() + .to_string(), + run_root: temp.path().join("run").display().to_string(), + state_root: state_root.display().to_string(), + site_state_root: direct_state_root.display().to_string(), + artifact_dir: direct_artifact.display().to_string(), + control_state_auth_token: "test-auth".to_string(), + dynamic_caps_token_verify_key_b64: String::new(), + storage_root: Some( + temp.path() + .join("storage") + .join("direct_local") + .display() + .to_string(), + ), + runtime_root: Some( + temp.path() + .join("runtime") + .join("direct_local") + .display() + .to_string(), + ), + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: None, + context: None, + observability_endpoint: None, + launch_env: BTreeMap::new(), + }; + + let runtime_plan = runtime_plan_for_site_from_controller_plan(&controller_plan, "vm_local") + .expect("vm runtime plan should derive from the managed site"); + assert_eq!(runtime_plan.site_id, "vm_local"); + assert_eq!(runtime_plan.kind, SiteKind::Vm); + assert_eq!(runtime_plan.artifact_dir, vm_artifact.display().to_string()); + assert_eq!( + runtime_plan.site_state_root, + vm_state_root.display().to_string() + ); + assert_eq!( + runtime_plan.local_router_control.as_deref(), + Some("unix:///tmp/vm-control.sock") + ); + assert_eq!(runtime_plan.router_identity_id, "/site/vm_local/router"); + assert_eq!(runtime_plan.router_mesh_port, Some(24001)); + assert_eq!( + runtime_plan + .launch_env + .get("AMBER_TEST") + .map(String::as_str), + Some("1") + ); + } } diff --git a/runtime/site-controller/src/default_runtime/compose_controller.rs b/runtime/site-controller/src/default_runtime/compose_controller.rs index 922384f5..b8201744 100644 --- a/runtime/site-controller/src/default_runtime/compose_controller.rs +++ b/runtime/site-controller/src/default_runtime/compose_controller.rs @@ -9,6 +9,7 @@ const COMPOSE_ROUTER_SERVICE_NAME: &str = "amber-router"; const COMPOSE_ROUTER_CONTROL_INIT_SERVICE_NAME: &str = "amber-router-control-init"; const COMPOSE_ROUTER_CONTROL_SOCKET_DIR: &str = "/amber/control"; const COMPOSE_ROUTER_CONTROL_VOLUME_NAME: &str = "amber-router-control"; +const COMPOSE_ROUTER_RUNTIME_GID: &str = "65532"; const DOCKER_SOCK_PATH: &str = "/var/run/docker.sock"; const COMPOSE_CONTROLLER_PLAN_PATH: &str = "/amber/site/state/site-controller-plan.json"; @@ -22,6 +23,13 @@ fn ensure_string_sequence_contains(sequence: &mut serde_yaml::Sequence, value: & sequence.push(yaml_string(value)); } +fn uses_service_network_mode(service: &serde_yaml::Mapping) -> bool { + service + .get(yaml_string("network_mode")) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|mode| mode.starts_with("service:")) +} + fn ensure_controller_environment( service: &mut serde_yaml::Mapping, plan: &SiteControllerPlan, @@ -150,6 +158,10 @@ pub fn inject_compose_site_controller( service.insert(yaml_string("image"), yaml_string(controller_image)); service.insert(yaml_string("user"), yaml_string("0:0")); + service.insert( + yaml_string("group_add"), + serde_yaml::Value::Sequence(vec![yaml_string(COMPOSE_ROUTER_RUNTIME_GID)]), + ); service.insert( yaml_string("healthcheck"), serde_yaml::to_value(json!({ @@ -170,14 +182,20 @@ pub fn inject_compose_site_controller( ); service.insert(yaml_string("restart"), yaml_string("unless-stopped")); - let extra_hosts = service - .entry(yaml_string("extra_hosts")) - .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) - .as_sequence_mut() - .ok_or_else(|| { - miette::miette!("compose site controller service has a non-sequence extra_hosts field") - })?; - ensure_string_sequence_contains(extra_hosts, "host.docker.internal:host-gateway"); + if uses_service_network_mode(service) { + service.remove(yaml_string("extra_hosts")); + } else { + let extra_hosts = service + .entry(yaml_string("extra_hosts")) + .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) + .as_sequence_mut() + .ok_or_else(|| { + miette::miette!( + "compose site controller service has a non-sequence extra_hosts field" + ) + })?; + ensure_string_sequence_contains(extra_hosts, "host.docker.internal:host-gateway"); + } let volumes = service .entry(yaml_string("volumes")) @@ -253,6 +271,55 @@ mod tests { } } + #[test] + fn inject_compose_site_controller_keeps_service_network_mode_compatible() { + let temp = tempfile::tempdir().expect("temp dir"); + let compose_path = temp.path().join("compose.yaml"); + fs::write( + &compose_path, + "services: + amber-site-controller: + image: __amber_internal/site-controller + network_mode: service:amber-site-controller-net + extra_hosts: + - host.docker.internal:host-gateway +", + ) + .expect("compose file should write"); + + let plan = test_plan(temp.path()); + let plan_path = temp.path().join("site-controller-plan.json"); + fs::write(&plan_path, "{}").expect("plan file should write"); + + inject_compose_site_controller( + temp.path(), + &plan, + &plan_path, + "ghcr.io/rdi-foundation/amber-site-controller:test", + ) + .expect("compose site controller injection should succeed"); + + let document = read_compose_document(&compose_path).expect("compose should parse"); + let service = document + .as_mapping() + .and_then(|root| root.get(yaml_string("services"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|services| services.get(yaml_string(SITE_CONTROLLER_SERVICE_NAME))) + .and_then(serde_yaml::Value::as_mapping) + .expect("site controller service should exist"); + assert_eq!( + service + .get(yaml_string("network_mode")) + .and_then(serde_yaml::Value::as_str), + Some("service:amber-site-controller-net"), + "late bootstrap must preserve the lowered sidecar sharing topology", + ); + assert!( + service.get(yaml_string("extra_hosts")).is_none(), + "late bootstrap must not add host mappings to service-networked controller workloads", + ); + } + #[test] fn inject_compose_site_controller_propagates_launch_env() { let temp = tempfile::tempdir().expect("temp dir"); diff --git a/runtime/site-controller/src/default_runtime/dynamic_routes.rs b/runtime/site-controller/src/default_runtime/dynamic_routes.rs index 134126dc..026c299b 100644 --- a/runtime/site-controller/src/default_runtime/dynamic_routes.rs +++ b/runtime/site-controller/src/default_runtime/dynamic_routes.rs @@ -58,16 +58,6 @@ pub(super) fn dynamic_proxy_export_route_id( )) } -pub(super) fn dynamic_input_route_mesh_protocol( - input: &DynamicInputRouteRecord, -) -> Result { - let protocol = input - .protocol - .parse::() - .map_err(|err| miette::miette!("invalid dynamic routed-input protocol: {err}"))?; - mesh_protocol(protocol) -} - pub(super) fn dynamic_input_direct_mesh_protocol( input: &DynamicInputDirectRecord, ) -> Result { @@ -78,23 +68,6 @@ pub(super) fn dynamic_input_direct_mesh_protocol( mesh_protocol(protocol) } -pub(super) fn dynamic_input_route_route_id( - input: &DynamicInputRouteRecord, - protocol: MeshProtocol, -) -> String { - match &input.target { - DynamicInputRouteTarget::ComponentProvide { provide } => { - component_route_id(&input.provider_component, provide, protocol) - } - } -} - -pub(super) fn dynamic_input_route_capability(input: &DynamicInputRouteRecord) -> String { - match &input.target { - DynamicInputRouteTarget::ComponentProvide { provide } => provide.clone(), - } -} - pub(super) fn dynamic_input_direct_route_id( input: &DynamicInputDirectRecord, protocol: MeshProtocol, @@ -116,24 +89,6 @@ pub(super) fn is_compose_component_sidecar_service(service_name: &str) -> bool { service_name.ends_with("-net") } -pub(super) fn dynamic_input_route_http_plugins( - input: &DynamicInputRouteRecord, - protocol: MeshProtocol, -) -> Vec { - dynamic_proxy_export_http_plugins( - &DynamicProxyExportRecord { - component_id: 0, - component: input.provider_component.clone(), - provide: dynamic_input_route_capability(input), - protocol: input.protocol.clone(), - capability_kind: input.capability_kind.clone(), - capability_profile: input.capability_profile.clone(), - target_port: 0, - }, - protocol, - ) -} - pub(super) fn dynamic_input_direct_http_plugins( input: &DynamicInputDirectRecord, protocol: MeshProtocol, @@ -161,21 +116,6 @@ pub(super) fn overlay_peer_addr_map_from_ports( .collect() } -pub(super) fn overlay_issuer_sets( - routed_inputs: &[DynamicInputRouteRecord], -) -> Result>> { - dynamic_route_issuer_grants(&[SiteControllerRuntimeChildRecord { - child_id: 0, - artifact_root: String::new(), - assigned_components: Vec::new(), - proxy_exports: BTreeMap::new(), - direct_inputs: Vec::new(), - routed_inputs: routed_inputs.to_vec(), - process_pid: None, - published: false, - }]) -} - pub(super) fn overlay_upsert_peer( peers: &mut Vec, peer_identities: &BTreeMap, @@ -194,54 +134,11 @@ pub(super) fn overlay_upsert_peer( Ok(()) } -pub(super) fn overlay_upsert_route(routes: &mut Vec, route: InboundRoute) { - if let Some(existing) = routes - .iter_mut() - .find(|existing| existing.route_id == route.route_id) - { - *existing = route; - } else { - routes.push(route); - } -} - -pub(super) fn routed_input_overlay_route( - input: &DynamicInputRouteRecord, - provider_peer_addr: &str, - allowed_issuers: Vec, -) -> Result { - let protocol = dynamic_input_route_mesh_protocol(input)?; - let (target_route_id, capability) = match &input.target { - DynamicInputRouteTarget::ComponentProvide { provide } => ( - component_route_id(&input.provider_component, provide, protocol), - provide.clone(), - ), - }; - Ok(InboundRoute { - route_id: dynamic_input_route_route_id(input, protocol), - capability: dynamic_input_route_capability(input), - capability_kind: Some(input.capability_kind.clone()), - capability_profile: input.capability_profile.clone(), - protocol, - http_plugins: dynamic_input_route_http_plugins(input, protocol), - target: InboundTarget::MeshForward { - peer_addr: provider_peer_addr.to_string(), - peer_id: input.provider_component.clone(), - route_id: target_route_id, - capability, - }, - allowed_issuers, - }) -} - pub(super) fn augment_route_overlay_payload( payload: &mut StoredRouteOverlayPayload, proxy_exports: &BTreeMap, - routed_inputs: &[DynamicInputRouteRecord], provider_peer_addrs: &BTreeMap, peer_identities: &BTreeMap, - allowed_issuers_by_route: Option<&BTreeMap>>, - skip_missing_providers: bool, ) -> Result<()> { for export in proxy_exports.values() { overlay_upsert_peer(&mut payload.peers, peer_identities, &export.component)?; @@ -262,138 +159,6 @@ pub(super) fn augment_route_overlay_payload( }, )?; - for input in routed_inputs { - let Some(provider_peer_addr) = provider_peer_addrs.get(&input.provider_component) else { - if skip_missing_providers { - continue; - } - return Err(miette::miette!( - "dynamic route overlay is missing a live peer address for {}", - input.provider_component - )); - }; - overlay_upsert_peer( - &mut payload.peers, - peer_identities, - &input.provider_component, - )?; - let route_id = - dynamic_input_route_route_id(input, dynamic_input_route_mesh_protocol(input)?); - let allowed_issuers = allowed_issuers_by_route - .and_then(|issuers| issuers.get(&route_id)) - .map(|issuers| issuers.iter().cloned().collect()) - .unwrap_or_default(); - overlay_upsert_route( - &mut payload.inbound_routes, - routed_input_overlay_route(input, provider_peer_addr, allowed_issuers)?, - ); - } - - Ok(()) -} - -pub(super) fn routed_input_router_peer_addr( - kind: SiteKind, - router_mesh_port: Option, -) -> Result { - let router_mesh_port = router_mesh_port.ok_or_else(|| { - miette::miette!("site {kind:?} is missing its router mesh port for routed child inputs") - })?; - Ok(match kind { - SiteKind::Direct | SiteKind::Vm => format!("127.0.0.1:{router_mesh_port}"), - SiteKind::Compose => format!("{COMPOSE_ROUTER_SERVICE_NAME}:{router_mesh_port}"), - SiteKind::Kubernetes => { - format!("{KUBERNETES_ROUTER_COMPONENT_NAME}:{router_mesh_port}") - } - }) -} - -pub(super) fn router_mesh_port_from_plan( - mesh_plan: &MeshProvisionPlan, - artifact_kind: &str, -) -> Result { - mesh_plan - .targets - .iter() - .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) - .map(|target| target.config.mesh_listen.port()) - .ok_or_else(|| { - miette::miette!("{artifact_kind} mesh provision plan is missing its router mesh target") - }) -} - -pub(super) fn rewrite_dynamic_routed_inputs( - mesh_plan: &mut MeshProvisionPlan, - routed_inputs: &[DynamicInputRouteRecord], - kind: SiteKind, - router_identity_id: &str, - router_mesh_port: Option, -) -> Result<()> { - if routed_inputs.is_empty() { - return Ok(()); - } - - let router_peer_addr = routed_input_router_peer_addr(kind, router_mesh_port)?; - for input in routed_inputs { - let protocol = dynamic_input_route_mesh_protocol(input)?; - let component_target = mesh_plan - .targets - .iter_mut() - .find(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && target.config.identity.id == input.component - }) - .ok_or_else(|| { - miette::miette!( - "dynamic routed input {}.{} is missing component {} in the mesh provision plan", - input.component, - input.slot, - input.component - ) - })?; - if !component_target - .config - .peers - .iter() - .any(|peer| peer.id == router_identity_id) - { - component_target - .config - .peers - .push(amber_mesh::MeshPeerTemplate { - id: router_identity_id.to_string(), - }); - } - - let route_id = dynamic_input_route_route_id(input, protocol); - let capability = dynamic_input_route_capability(input); - let mut matched = false; - for route in component_target - .config - .outbound - .iter_mut() - .filter(|route| route.slot == input.slot) - { - matched = true; - route.route_id = route_id.clone(); - route.protocol = protocol; - route.peer_addr = router_peer_addr.clone(); - route.peer_id = router_identity_id.to_string(); - route.capability = capability.clone(); - route.capability_kind = Some(input.capability_kind.clone()); - route.capability_profile = input.capability_profile.clone(); - route.http_plugins = dynamic_input_route_http_plugins(input, protocol); - } - if !matched { - return Err(miette::miette!( - "dynamic routed input {}.{} is missing an outbound route in the mesh provision \ - plan", - input.component, - input.slot - )); - } - } - Ok(()) } @@ -479,28 +244,6 @@ pub(super) fn rewrite_dynamic_direct_inputs( Ok(()) } -pub(super) fn rewrite_dynamic_routed_inputs_in_artifact( - artifact_root: &Path, - routed_inputs: &[DynamicInputRouteRecord], - kind: SiteKind, - router_identity_id: &str, - router_mesh_port: Option, -) -> Result<()> { - if routed_inputs.is_empty() { - return Ok(()); - } - let path = artifact_root.join("mesh-provision-plan.json"); - let mut mesh_plan: MeshProvisionPlan = read_json(&path, "mesh provision plan")?; - rewrite_dynamic_routed_inputs( - &mut mesh_plan, - routed_inputs, - kind, - router_identity_id, - router_mesh_port, - )?; - write_json(&path, &mesh_plan) -} - pub(super) fn rewrite_dynamic_direct_inputs_in_artifact( artifact_root: &Path, direct_inputs: &[DynamicInputDirectRecord], @@ -570,41 +313,10 @@ pub(super) fn build_filesystem_route_overlay_base( }) } -pub(super) fn write_direct_vm_startup_route_overlay_payload( - artifact_root: &Path, - _artifact_kind: &str, - routed_inputs: &[DynamicInputRouteRecord], - provider_peer_addrs: &BTreeMap, - existing_site_peer_identities: &BTreeMap, -) -> Result<()> { - if routed_inputs.is_empty() { - return Ok(()); - } - let allowed_issuers = overlay_issuer_sets(routed_inputs)?; - let mut payload = StoredRouteOverlayPayload { - peers: Vec::new(), - inbound_routes: Vec::new(), - }; - augment_route_overlay_payload( - &mut payload, - &BTreeMap::new(), - routed_inputs, - provider_peer_addrs, - existing_site_peer_identities, - Some(&allowed_issuers), - true, - )?; - if payload.inbound_routes.is_empty() { - return Ok(()); - } - write_dynamic_route_overlay_payload(artifact_root, &payload) -} - pub(super) fn write_direct_vm_live_route_overlay_payload( artifact_root: &Path, assigned_components: &[String], proxy_exports: &BTreeMap, - routed_inputs: &[DynamicInputRouteRecord], provider_peer_addrs: &BTreeMap, peer_identities: &BTreeMap, ) -> Result<()> { @@ -617,11 +329,8 @@ pub(super) fn write_direct_vm_live_route_overlay_payload( augment_route_overlay_payload( &mut payload, proxy_exports, - routed_inputs, provider_peer_addrs, peer_identities, - None, - false, )?; write_dynamic_route_overlay_payload(artifact_root, &payload) } @@ -947,6 +656,11 @@ pub(super) fn dynamic_child_route_overlay_id( pub(super) fn site_router_control_endpoint( plan: &SiteControllerRuntimePlan, ) -> Result { + if plan.kind == SiteKind::Kubernetes + && let Some(raw) = plan.local_router_control.as_deref() + { + return parse_control_endpoint(raw); + } let state_path = Path::new(&plan.site_state_root).join("manager-state.json"); if state_path.is_file() { let state: SiteManagerState = read_json(&state_path, "site manager state")?; @@ -954,9 +668,6 @@ pub(super) fn site_router_control_endpoint( return parse_control_endpoint(&raw); } } - if let Some(raw) = plan.local_router_control.as_deref() { - return parse_control_endpoint(raw); - } match plan.kind { SiteKind::Direct => Ok(ControlEndpoint::Unix(direct_current_control_socket_path( @@ -972,6 +683,23 @@ pub(super) fn site_router_control_endpoint( } } +async fn site_router_overlay_control_endpoint_with_timeout( + plan: &SiteControllerRuntimePlan, + timeout: Duration, +) -> Result { + if plan.kind == SiteKind::Kubernetes { + wait_for_kubernetes_site_router_ready(plan, timeout).await?; + } + site_router_control_endpoint(plan) +} + +async fn site_router_overlay_control_endpoint( + plan: &SiteControllerRuntimePlan, +) -> Result { + site_router_overlay_control_endpoint_with_timeout(plan, site_ready_timeout_for_kind(plan.kind)) + .await +} + pub(super) fn child_router_overlay_payload( plan: &SiteControllerRuntimePlan, artifact_root: &Path, @@ -1094,105 +822,120 @@ pub(super) fn child_overlay_runtime_root( } } -pub(super) fn dynamic_route_issuer_grants( - children: &[SiteControllerRuntimeChildRecord], -) -> Result>> { - let mut issuers_by_route_id = BTreeMap::>::new(); - for child in children { - for input in &child.routed_inputs { - let route_id = - dynamic_input_route_route_id(input, dynamic_input_route_mesh_protocol(input)?); - issuers_by_route_id - .entry(route_id) - .or_default() - .insert(input.component.clone()); - } +pub(super) const SITE_CONTROLLER_INTERNAL_OVERLAY_ID: &str = "framework-site-controller"; + +fn local_site_controller_runtime( + live_components: &BTreeMap, +) -> Result> { + let mut controllers = live_components.values().filter(|runtime| { + runtime.mesh_config.inbound.iter().any(|route| { + route.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + }) + }); + let Some(controller) = controllers.next() else { + return Ok(None); + }; + if controllers.next().is_some() { + return Err(miette::miette!( + "dynamic site controller overlay found multiple local framework.component controller \ + runtimes" + )); } - Ok(issuers_by_route_id) + Ok(Some(controller)) } -pub(super) fn load_published_component_peers( - plan: &SiteControllerRuntimePlan, +fn dynamic_site_controller_overlay_payload( published_children: &[SiteControllerRuntimeChildRecord], -) -> Result> { - let mut component_peers = BTreeMap::new(); + live_components: &BTreeMap, +) -> Result> { + let Some(controller_runtime) = local_site_controller_runtime(live_components)? else { + return Ok(None); + }; + + let mut routes_by_id = BTreeMap::::new(); + let mut issuer_ids = BTreeSet::::new(); for child in published_children { - if child.assigned_components.is_empty() { - continue; - } - let artifact_root = Path::new(&child.artifact_root); - let runtime_root = child_overlay_runtime_root(plan, child); - let provision: MeshProvisionPlan = read_json( - &artifact_root.join("mesh-provision-plan.json"), - "mesh provision plan", - )?; - for component in &child.assigned_components { - let target = provision - .targets - .iter() - .find(|target| { - matches!(target.kind, MeshProvisionTargetKind::Component) - && target.config.identity.id == *component - }) - .ok_or_else(|| { - miette::miette!( - "published child {} is missing component {} in its mesh provision plan", - child.child_id, - component - ) - })?; - let identity = match &target.output { - MeshProvisionOutput::Filesystem { dir } => read_json( - &runtime_root.join(dir).join(MESH_IDENTITY_FILENAME), - "mesh identity", - )?, - MeshProvisionOutput::KubernetesSecret { name, namespace } => { - load_kubernetes_mesh_identity_secret(plan, name, namespace.as_deref())? + for route in &child.controller_routes { + if route.allowed_issuers.is_empty() { + continue; + } + issuer_ids.extend(route.allowed_issuers.iter().cloned()); + if let Some(existing) = routes_by_id.get_mut(&route.route_id) { + if !controller_overlay_routes_compatible(existing, route) { + return Err(miette::miette!( + "dynamic controller overlay route {} is defined with incompatible route \ + metadata", + route.route_id + )); } - }; - component_peers.insert( - component.clone(), - MeshPeer { - id: identity.id.clone(), - public_key: identity.public_key().into_diagnostic()?, - }, - ); + existing + .allowed_issuers + .extend(route.allowed_issuers.iter().cloned()); + existing.allowed_issuers.sort(); + existing.allowed_issuers.dedup(); + } else { + let mut route = route.clone(); + route.allowed_issuers.sort(); + route.allowed_issuers.dedup(); + routes_by_id.insert(route.route_id.clone(), route); + } } } - Ok(component_peers) -} + if routes_by_id.is_empty() { + return Ok(None); + } -pub(super) fn apply_dynamic_route_issuer_grants( - peers: &mut Vec, - inbound_routes: &mut [InboundRoute], - issuers_by_route_id: &BTreeMap>, - component_peers: &BTreeMap, -) -> Result<()> { - let mut known_peer_ids = peers + let static_peer_ids = controller_runtime + .mesh_config + .peers .iter() - .map(|peer| peer.id.clone()) + .map(|peer| peer.id.as_str()) .collect::>(); - for route in inbound_routes { - let Some(issuers) = issuers_by_route_id.get(&route.route_id) else { + let mut peers = Vec::new(); + for issuer_id in issuer_ids { + if issuer_id == controller_runtime.mesh_config.identity.id + || static_peer_ids.contains(issuer_id.as_str()) + { continue; - }; - route.allowed_issuers = issuers.iter().cloned().collect(); - for issuer in issuers { - if known_peer_ids.contains(issuer) { - continue; - } - let peer = component_peers.get(issuer).ok_or_else(|| { - miette::miette!( - "dynamic route {} references published issuer {} with no live mesh peer", - route.route_id, - issuer - ) - })?; - peers.push(peer.clone()); - known_peer_ids.insert(issuer.clone()); } + let runtime = live_components.get(&issuer_id).ok_or_else(|| { + miette::miette!( + "dynamic controller overlay route grants issuer {} but that peer is not live on \ + site {}", + issuer_id, + controller_runtime.moniker + ) + })?; + peers.push(MeshPeer { + id: runtime.mesh_config.identity.id.clone(), + public_key: runtime.mesh_config.identity.public_key, + }); } - Ok(()) + + Ok(Some(StoredRouteOverlayPayload { + peers, + inbound_routes: routes_by_id.into_values().collect(), + })) +} + +fn dynamic_site_controller_overlay_required( + published_children: &[SiteControllerRuntimeChildRecord], +) -> bool { + published_children.iter().any(|child| { + child + .controller_routes + .iter() + .any(|route| !route.allowed_issuers.is_empty()) + }) +} + +fn controller_overlay_routes_compatible(left: &InboundRoute, right: &InboundRoute) -> bool { + left.capability == right.capability + && left.capability_kind == right.capability_kind + && left.capability_profile == right.capability_profile + && left.protocol == right.protocol + && left.http_plugins == right.http_plugins + && left.target == right.target } pub(super) fn dynamic_direct_input_overlay_id(component: &str) -> String { @@ -1221,7 +964,7 @@ pub(super) fn dynamic_direct_input_grants( Ok(grants) } -pub(super) async fn reconcile_dynamic_direct_input_overlays( +pub(super) async fn reconcile_dynamic_site_controller_overlay( app: &SiteControllerRuntimeApp, ) -> Result<()> { let published_children = { @@ -1234,8 +977,66 @@ pub(super) async fn reconcile_dynamic_direct_input_overlays( .collect::>() }; let live_components = collect_live_component_runtime_metadata(&app.plan)?; + let Some(controller_runtime) = local_site_controller_runtime(&live_components)? else { + if !dynamic_site_controller_overlay_required(&published_children) { + return Ok(()); + } + return Err(miette::miette!( + "site {} has published dynamic children but no live framework.component controller \ + component", + app.plan.site_id + )); + }; + let Some(control_endpoint) = controller_runtime.control_endpoint.as_ref() else { + return Ok(()); + }; + let Some(overlay) = + dynamic_site_controller_overlay_payload(&published_children, &live_components)? + else { + revoke_route_overlay_with_retry( + control_endpoint, + SITE_CONTROLLER_INTERNAL_OVERLAY_ID, + Duration::from_secs(30), + ) + .await?; + return Ok(()); + }; + apply_route_overlay_with_retry( + control_endpoint, + SITE_CONTROLLER_INTERNAL_OVERLAY_ID, + &overlay.peers, + &overlay.inbound_routes, + Duration::from_secs(30), + ) + .await +} + +pub(super) async fn reconcile_dynamic_direct_input_overlays( + app: &SiteControllerRuntimeApp, +) -> Result<()> { + let (published_children, previous_overlay_providers) = { + let state = app.state.lock().await; + ( + state + .children + .values() + .filter(|child| child.published) + .cloned() + .collect::>(), + state.direct_input_overlay_providers.clone(), + ) + }; + let live_components = collect_live_component_runtime_metadata(&app.plan)?; let grants = dynamic_direct_input_grants(&published_children)?; - for (component, runtime) in &live_components { + let current_overlay_providers = grants.keys().cloned().collect::>(); + let overlay_providers = current_overlay_providers + .union(&previous_overlay_providers) + .cloned() + .collect::>(); + for component in &overlay_providers { + let Some(runtime) = live_components.get(component) else { + continue; + }; let Some(control_endpoint) = runtime.control_endpoint.as_ref() else { continue; }; @@ -1292,6 +1093,11 @@ pub(super) async fn reconcile_dynamic_direct_input_overlays( ) .await?; } + let mut state = app.state.lock().await; + if state.direct_input_overlay_providers != current_overlay_providers { + state.direct_input_overlay_providers = current_overlay_providers; + write_json(&app.state_path, &*state)?; + } Ok(()) } @@ -1323,24 +1129,16 @@ pub(super) async fn reconcile_dynamic_site_router_overlays_for_children( if overlay_children.is_empty() { return Ok(()); } - - let issuers_by_route_id = dynamic_route_issuer_grants(issuer_children)?; - let component_peers = load_published_component_peers(&app.plan, issuer_children)?; + let endpoint = site_router_overlay_control_endpoint(&app.plan).await?; + let _ = issuer_children; for child in overlay_children { let artifact_root = Path::new(&child.artifact_root); let runtime_root = child_overlay_runtime_root(&app.plan, child); - let (mut peers, mut inbound_routes) = + let (peers, inbound_routes) = child_router_overlay_payload(&app.plan, artifact_root, &runtime_root)?; - apply_dynamic_route_issuer_grants( - &mut peers, - &mut inbound_routes, - &issuers_by_route_id, - &component_peers, - )?; if inbound_routes.is_empty() { continue; } - let endpoint = site_router_control_endpoint(&app.plan)?; apply_route_overlay_with_retry( &endpoint, &dynamic_child_route_overlay_id(&app.plan, child.child_id), @@ -1358,13 +1156,13 @@ pub(super) async fn apply_dynamic_site_router_overlay( plan: &SiteControllerRuntimePlan, child: &SiteControllerRuntimeChildRecord, ) -> Result<()> { + let endpoint = site_router_overlay_control_endpoint(plan).await?; let artifact_root = Path::new(&child.artifact_root); let runtime_root = child_overlay_runtime_root(plan, child); let (peers, inbound_routes) = child_router_overlay_payload(plan, artifact_root, &runtime_root)?; if inbound_routes.is_empty() { return Ok(()); } - let endpoint = site_router_control_endpoint(plan)?; apply_route_overlay_with_retry( &endpoint, &dynamic_child_route_overlay_id(plan, child.child_id), @@ -1379,7 +1177,7 @@ pub(super) async fn revoke_dynamic_site_router_overlay( plan: &SiteControllerRuntimePlan, child: &SiteControllerRuntimeChildRecord, ) -> Result<()> { - let endpoint = site_router_control_endpoint(plan)?; + let endpoint = site_router_overlay_control_endpoint(plan).await?; revoke_route_overlay_with_retry( &endpoint, &dynamic_child_route_overlay_id(plan, child.child_id), @@ -1390,12 +1188,318 @@ pub(super) async fn revoke_dynamic_site_router_overlay( #[cfg(test)] mod direct_input_tests { - use std::fs; + use std::{ + fs, + net::SocketAddr, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration as StdDuration, Instant as StdInstant}, + }; use amber_mesh::MeshConfigTemplate; + use axum::response::IntoResponse; use super::*; + async fn spawn_mock_router_mesh_listener() -> (SocketAddr, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("mock router mesh listener should bind"); + let addr = listener + .local_addr() + .expect("mock router mesh listener addr should resolve"); + let handle = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + drop(stream); + } + }); + (addr, handle) + } + + async fn spawn_gated_router_control_listener( + ready: Arc, + saw_mutation: Arc, + mutated_before_ready: Arc, + ) -> (SocketAddr, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("mock router control listener should bind"); + let addr = listener + .local_addr() + .expect("mock router control listener addr should resolve"); + let router = axum::Router::new() + .route( + "/identity", + axum::routing::get({ + let ready = ready.clone(); + move || { + let ready = ready.clone(); + async move { + if ready.load(Ordering::SeqCst) { + axum::Json(MeshIdentityPublic { + id: "/site/kind_local/router".to_string(), + public_key: [9u8; 32], + mesh_scope: Some("test-mesh".to_string()), + }) + .into_response() + } else { + axum::http::StatusCode::SERVICE_UNAVAILABLE.into_response() + } + } + } + }), + ) + .route( + "/overlays/{overlay_id}", + axum::routing::put({ + let ready = ready.clone(); + let saw_mutation = saw_mutation.clone(); + let mutated_before_ready = mutated_before_ready.clone(); + move || { + let ready = ready.clone(); + let saw_mutation = saw_mutation.clone(); + let mutated_before_ready = mutated_before_ready.clone(); + async move { + saw_mutation.store(true, Ordering::SeqCst); + if !ready.load(Ordering::SeqCst) { + mutated_before_ready.store(true, Ordering::SeqCst); + } + axum::http::StatusCode::NO_CONTENT + } + } + }) + .delete({ + let ready = ready.clone(); + let saw_mutation = saw_mutation.clone(); + let mutated_before_ready = mutated_before_ready.clone(); + move || { + let ready = ready.clone(); + let saw_mutation = saw_mutation.clone(); + let mutated_before_ready = mutated_before_ready.clone(); + async move { + saw_mutation.store(true, Ordering::SeqCst); + if !ready.load(Ordering::SeqCst) { + mutated_before_ready.store(true, Ordering::SeqCst); + } + axum::http::StatusCode::NO_CONTENT + } + } + }), + ); + let handle = tokio::spawn(async move { + axum::serve(listener, router.into_make_service()) + .await + .expect("mock router control server should run"); + }); + (addr, handle) + } + + fn kubernetes_overlay_test_plan( + temp: &tempfile::TempDir, + control_addr: SocketAddr, + mesh_addr: SocketAddr, + ) -> SiteControllerRuntimePlan { + let artifact_dir = temp.path().join("artifact"); + let site_state_root = temp.path().join("state").join("kind_local"); + fs::create_dir_all(&artifact_dir).expect("artifact dir should exist"); + fs::create_dir_all(&site_state_root).expect("site state root should exist"); + SiteControllerRuntimePlan { + schema: "amber.run.site_controller_runtime_plan".to_string(), + version: 1, + run_id: "test-run".to_string(), + mesh_scope: "test-mesh".to_string(), + run_root: temp.path().display().to_string(), + site_id: "kind_local".to_string(), + kind: SiteKind::Kubernetes, + router_identity_id: "/site/kind_local/router".to_string(), + local_router_control: Some(control_addr.to_string()), + artifact_dir: artifact_dir.display().to_string(), + site_state_root: site_state_root.display().to_string(), + listen_addr: "127.0.0.1:32000".parse().expect("listen addr"), + storage_root: None, + runtime_root: None, + router_mesh_port: Some(mesh_addr.port()), + compose_project: None, + kubernetes_namespace: Some("amber-test-kind-local".to_string()), + context: None, + observability_endpoint: None, + launch_env: BTreeMap::new(), + } + } + + #[test] + fn compose_site_router_control_endpoint_prefers_manager_state_volume_socket() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let site_state_root = temp.path().join("state").join("compose_local"); + fs::create_dir_all(&site_state_root).expect("site state root should exist"); + write_json( + &site_state_root.join("manager-state.json"), + &serde_json::json!({ + "schema": "amber.run.site_manager_state", + "version": 1, + "run_id": "test-run", + "site_id": "compose_local", + "kind": "compose", + "status": "running", + "artifact_dir": temp.path().join("artifact").display().to_string(), + "supervisor_pid": 1u32, + "router_control": "volume://demo_amber-router-control/router-control.sock", + "router_mesh_addr": "127.0.0.1:24000", + }), + ) + .expect("manager state should write"); + let plan = SiteControllerRuntimePlan { + schema: "amber.run.site_controller_runtime_plan".to_string(), + version: 1, + run_id: "test-run".to_string(), + mesh_scope: "test-mesh".to_string(), + run_root: temp.path().display().to_string(), + site_id: "compose_local".to_string(), + kind: SiteKind::Compose, + router_identity_id: "/site/compose_local/router".to_string(), + local_router_control: Some("unix:///amber/control/router-control.sock".to_string()), + artifact_dir: temp.path().join("artifact").display().to_string(), + site_state_root: site_state_root.display().to_string(), + listen_addr: "127.0.0.1:32000".parse().expect("listen addr"), + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24000), + compose_project: Some("demo".to_string()), + kubernetes_namespace: None, + context: None, + observability_endpoint: None, + launch_env: BTreeMap::new(), + }; + + assert!(matches!( + site_router_control_endpoint(&plan).expect("router control endpoint should resolve"), + ControlEndpoint::VolumeSocket { volume, socket_path } + if volume == "demo_amber-router-control" && socket_path == "/router-control.sock" + )); + } + + #[test] + fn site_router_control_endpoint_prefers_local_embedded_target_over_manager_state() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let site_state_root = temp.path().join("state").join("kind_local"); + fs::create_dir_all(&site_state_root).expect("site state root should exist"); + write_json( + &site_state_root.join("manager-state.json"), + &serde_json::json!({ + "schema": "amber.run.site_manager_state", + "version": 1, + "run_id": "test-run", + "site_id": "kind_local", + "kind": "kubernetes", + "status": "running", + "artifact_dir": temp.path().join("artifact").display().to_string(), + "supervisor_pid": 1u32, + "router_control": "127.0.0.1:9", + "router_mesh_addr": "127.0.0.1:9", + }), + ) + .expect("manager state should write"); + let plan = SiteControllerRuntimePlan { + schema: "amber.run.site_controller_runtime_plan".to_string(), + version: 1, + run_id: "test-run".to_string(), + mesh_scope: "test-mesh".to_string(), + run_root: temp.path().display().to_string(), + site_id: "kind_local".to_string(), + kind: SiteKind::Kubernetes, + router_identity_id: "/site/kind_local/router".to_string(), + local_router_control: Some("amber-router:24100".to_string()), + artifact_dir: temp.path().join("artifact").display().to_string(), + site_state_root: site_state_root.display().to_string(), + listen_addr: "127.0.0.1:32000".parse().expect("listen addr"), + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: Some("amber-test-kind-local".to_string()), + context: None, + observability_endpoint: None, + launch_env: BTreeMap::new(), + }; + + assert!(matches!( + site_router_control_endpoint(&plan).expect("router control endpoint should resolve"), + ControlEndpoint::Tcp(addr) if addr == "amber-router:24100" + )); + } + + #[tokio::test] + async fn probe_kubernetes_router_control_ready_accepts_mock_identity_response() { + let ready = Arc::new(AtomicBool::new(true)); + let (control_addr, control_handle) = spawn_gated_router_control_listener( + ready, + Arc::new(AtomicBool::new(false)), + Arc::new(AtomicBool::new(false)), + ) + .await; + + assert!( + probe_kubernetes_router_control_ready( + &control_addr.to_string(), + StdDuration::from_secs(1) + ) + .await + .expect("router control probe should succeed"), + "the mock router control server should satisfy the readiness probe", + ); + control_handle.abort(); + } + + #[tokio::test] + async fn router_mesh_listener_ready_target_accepts_mock_listener() { + let (mesh_addr, mesh_handle) = spawn_mock_router_mesh_listener().await; + assert!( + router_mesh_listener_ready_target(&mesh_addr.to_string(), StdDuration::from_secs(1)) + .await, + "the mock mesh listener should satisfy the mesh readiness probe", + ); + mesh_handle.abort(); + } + + #[tokio::test] + async fn site_router_overlay_control_endpoint_waits_for_kubernetes_router_readiness() { + let temp = tempfile::tempdir().expect("tempdir should create"); + let ready = Arc::new(AtomicBool::new(false)); + let saw_mutation = Arc::new(AtomicBool::new(false)); + let mutated_before_ready = Arc::new(AtomicBool::new(false)); + let (mesh_addr, mesh_handle) = spawn_mock_router_mesh_listener().await; + let (control_addr, control_handle) = + spawn_gated_router_control_listener(ready.clone(), saw_mutation, mutated_before_ready) + .await; + let plan = kubernetes_overlay_test_plan(&temp, control_addr, mesh_addr); + + let gate = ready.clone(); + let gate_handle = tokio::spawn(async move { + tokio::time::sleep(StdDuration::from_millis(350)).await; + gate.store(true, Ordering::SeqCst); + }); + + let start = StdInstant::now(); + let endpoint = + site_router_overlay_control_endpoint_with_timeout(&plan, StdDuration::from_secs(6)) + .await + .expect("overlay endpoint should wait for router readiness and then resolve"); + + assert!( + start.elapsed() >= StdDuration::from_millis(300), + "overlay endpoint resolution should wait until the Kubernetes router is ready", + ); + assert!(matches!( + endpoint, + ControlEndpoint::Tcp(addr) if addr == control_addr.to_string() + )); + gate_handle.abort(); + control_handle.abort(); + mesh_handle.abort(); + } + #[test] fn rewrite_dynamic_direct_inputs_points_child_at_provider_sidecar() { let temp = tempfile::tempdir().expect("tempdir"); @@ -1500,46 +1604,303 @@ mod direct_input_tests { } #[test] - fn dynamic_route_issuer_grants_include_component_provide_inputs() { - let issuers = dynamic_route_issuer_grants(&[SiteControllerRuntimeChildRecord { + fn dynamic_direct_input_overlay_id_is_path_safe_for_component_monikers() { + let overlay_id = dynamic_direct_input_overlay_id("/source"); + assert_eq!( + overlay_id, "framework-direct-inputs:L3NvdXJjZQ", + "overlay ids should encode component monikers so router control paths stay \ + single-segment", + ); + assert!( + !overlay_id.contains('/'), + "overlay ids must not contain path separators: {overlay_id}", + ); + } + + #[test] + fn dynamic_direct_input_overlay_targets_only_current_and_previous_providers() { + let current = BTreeSet::from(["/provider".to_string()]); + let previous = BTreeSet::from(["/provider".to_string(), "/stale".to_string()]); + + let overlay_providers = current.union(&previous).cloned().collect::>(); + + assert_eq!( + overlay_providers, + BTreeSet::from(["/provider".to_string(), "/stale".to_string()]), + "reconciliation should only touch providers with current grants or previously applied \ + overlays", + ); + } + + #[test] + fn dynamic_site_controller_overlay_uses_child_controller_route_grants() { + let internal_route_id = component_route_id( + "/__amber_internal_framework_component_controller/site-a", + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + MeshProtocol::Http, + ); + let grant_route_id = component_route_id( + "/__amber_internal_framework_component_controller/site-a", + "__amber_internal_framework_component__site_site-a__authority_root", + MeshProtocol::Http, + ); + let live_components = BTreeMap::from([ + ( + "/__amber_internal_framework_component_controller/site-a".to_string(), + LiveComponentRuntimeMetadata { + moniker: "/__amber_internal_framework_component_controller/site-a".to_string(), + router_reachable_mesh_addr: "127.0.0.1:24000".to_string(), + component_reachable_mesh_addr: "127.0.0.1:24000".to_string(), + control_endpoint: Some(ControlEndpoint::Unix("/tmp/controller.sock".into())), + mesh_config: MeshConfigPublic { + identity: MeshIdentityPublic { + id: "/__amber_internal_framework_component_controller/site-a" + .to_string(), + public_key: [1; 32], + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:23000".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: None, + control_allow: None, + peers: vec![MeshPeer { + id: "/static".to_string(), + public_key: [2; 32], + }], + inbound: vec![InboundRoute { + route_id: internal_route_id.clone(), + capability: + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + .to_string(), + capability_kind: None, + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: InboundTarget::Local { port: 8080 }, + allowed_issuers: vec![ + "/site/router".to_string(), + "/static".to_string(), + ], + }], + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + }, + ), + ( + "/job-dynamic".to_string(), + LiveComponentRuntimeMetadata { + moniker: "/job-dynamic".to_string(), + router_reachable_mesh_addr: "127.0.0.1:24001".to_string(), + component_reachable_mesh_addr: "127.0.0.1:24001".to_string(), + control_endpoint: Some(ControlEndpoint::Unix("/tmp/job.sock".into())), + mesh_config: MeshConfigPublic { + identity: MeshIdentityPublic { + id: "/job-dynamic".to_string(), + public_key: [3; 32], + mesh_scope: None, + }, + mesh_listen: "127.0.0.1:23001".parse().expect("mesh listen"), + control_listen: None, + dynamic_caps_listen: Some( + "127.0.0.1:19001".parse().expect("dynamic caps listen"), + ), + control_allow: None, + peers: Vec::new(), + inbound: Vec::new(), + outbound: Vec::new(), + transport: amber_mesh::TransportConfig::NoiseIk {}, + }, + }, + ), + ]); + + let overlay = dynamic_site_controller_overlay_payload( + &[SiteControllerRuntimeChildRecord { + child_id: 7, + artifact_root: "/tmp/child".to_string(), + assigned_components: vec!["/job-dynamic".to_string()], + controller_routes: vec![InboundRoute { + route_id: grant_route_id.clone(), + capability: "__amber_internal_framework_component__site_site-a__authority_root" + .to_string(), + capability_kind: Some("framework.component".to_string()), + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: InboundTarget::Local { port: 8080 }, + allowed_issuers: vec!["/job-dynamic".to_string()], + }], + proxy_exports: BTreeMap::new(), + direct_inputs: Vec::new(), + process_pid: None, + published: true, + }], + &live_components, + ) + .expect("overlay payload should build") + .expect("dynamic child should require an overlay"); + + assert_eq!(overlay.peers.len(), 1); + assert_eq!(overlay.peers[0].id, "/job-dynamic"); + assert_eq!(overlay.peers[0].public_key, [3; 32]); + assert_eq!(overlay.inbound_routes.len(), 1); + assert_eq!(overlay.inbound_routes[0].route_id, grant_route_id); + assert_eq!( + overlay.inbound_routes[0].allowed_issuers, + vec!["/job-dynamic".to_string()], + "the overlay should contribute only the dynamic child issuer recorded for this \ + grant-specific controller route", + ); + } + + #[test] + fn dynamic_site_controller_overlay_is_required_only_for_granted_controller_routes() { + let child = |controller_routes| SiteControllerRuntimeChildRecord { child_id: 7, artifact_root: "/tmp/child".to_string(), - assigned_components: vec!["/sibling".to_string()], + assigned_components: vec!["/job-dynamic".to_string()], + controller_routes, proxy_exports: BTreeMap::new(), direct_inputs: Vec::new(), - routed_inputs: vec![DynamicInputRouteRecord { - component: "/sibling".to_string(), - slot: "upstream".to_string(), - provider_component: "/provider".to_string(), - protocol: "http".to_string(), - capability_kind: "http".to_string(), - capability_profile: None, - target: DynamicInputRouteTarget::ComponentProvide { - provide: "http".to_string(), - }, - }], process_pid: None, published: true, - }]) - .expect("component-provide routed inputs should produce issuer grants"); + }; - assert_eq!( - issuers.get("component:/provider:http:http"), - Some(&BTreeSet::from(["/sibling".to_string()])) + assert!( + !dynamic_site_controller_overlay_required(&[child(Vec::new())]), + "VM sites controlled from a direct site can publish ordinary dynamic children without \ + a local site-controller component" + ); + assert!( + !dynamic_site_controller_overlay_required(&[child(vec![InboundRoute { + route_id: "empty".to_string(), + capability: "component".to_string(), + capability_kind: Some("framework.component".to_string()), + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: InboundTarget::Local { port: 8080 }, + allowed_issuers: Vec::new(), + }])]), + "routes with no dynamic issuers do not require a controller overlay" + ); + assert!( + dynamic_site_controller_overlay_required(&[child(vec![InboundRoute { + route_id: "granted".to_string(), + capability: "component".to_string(), + capability_kind: Some("framework.component".to_string()), + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: InboundTarget::Local { port: 8080 }, + allowed_issuers: vec!["/job-dynamic".to_string()], + }])]), + "controller overlays are still required when a dynamic child receives a granted \ + framework.component route" ); } #[test] - fn dynamic_direct_input_overlay_id_is_path_safe_for_component_monikers() { - let overlay_id = dynamic_direct_input_overlay_id("/source"); + fn project_dynamic_direct_router_surface_creates_child_control_aliases() { + let temp = tempfile::tempdir().expect("tempdir should be created"); + let site_artifact = temp.path().join("site-artifact"); + let site_state_root = temp.path().join("state").join("direct_local"); + let child_artifact = site_state_root + .join("framework-component") + .join("children") + .join("1") + .join("artifact"); + fs::create_dir_all(site_artifact.join(".amber").join("control")) + .expect("site artifact control dir should exist"); + fs::create_dir_all(child_artifact.join(".amber")).expect("child artifact dir should exist"); + + let site_current = super::super::direct_current_control_socket_path(&site_artifact); + if let Some(parent) = site_current.parent() { + fs::create_dir_all(parent).expect("site current control dir should exist"); + } + let site_runtime = temp.path().join("router-runtime.sock"); + fs::write(&site_runtime, []).expect("fake router runtime socket placeholder should exist"); + super::super::ensure_direct_control_socket_link( + &site_current, + &site_runtime, + "site current control symlink", + ) + .expect("site current control symlink should be created"); + super::super::ensure_direct_control_socket_link( + &site_artifact + .join(".amber") + .join("control") + .join("router-control.sock"), + &site_current, + "site artifact control symlink", + ) + .expect("site artifact control symlink should be created"); + write_json( + &super::super::direct_runtime_state_path(&child_artifact), + &super::super::DirectRuntimeState::default(), + ) + .expect("child direct runtime state should be written"); + + let plan = runtime_api::SiteControllerRuntimePlan { + schema: "test".to_string(), + version: 1, + run_id: "run-123".to_string(), + mesh_scope: "mesh".to_string(), + run_root: temp.path().display().to_string(), + site_id: "direct_local".to_string(), + kind: SiteKind::Direct, + router_identity_id: "/site/direct_local/router".to_string(), + local_router_control: None, + artifact_dir: site_artifact.display().to_string(), + site_state_root: site_state_root.display().to_string(), + listen_addr: "127.0.0.1:32000".parse().expect("listen addr"), + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: None, + context: None, + observability_endpoint: None, + launch_env: BTreeMap::new(), + }; + let child = SiteControllerRuntimeChildRecord { + child_id: 1, + artifact_root: child_artifact.display().to_string(), + assigned_components: vec!["/job-1".to_string()], + controller_routes: Vec::new(), + proxy_exports: BTreeMap::new(), + direct_inputs: Vec::new(), + process_pid: None, + published: true, + }; + + project_dynamic_direct_router_surface(&plan, &child) + .expect("direct child router surface should project"); + + let state: super::super::DirectRuntimeState = read_json( + &super::super::direct_runtime_state_path(&child_artifact), + "direct runtime state", + ) + .expect("projected direct runtime state should be readable"); + assert_eq!(state.router_mesh_port, Some(24000)); + + let child_current = super::super::direct_current_control_socket_path(&child_artifact); assert_eq!( - overlay_id, "framework-direct-inputs:L3NvdXJjZQ", - "overlay ids should encode component monikers so router control paths stay \ - single-segment", + fs::read_link(&child_current).expect("child current control alias should exist"), + site_current, + "child current alias should point at the site current control alias", ); - assert!( - !overlay_id.contains('/'), - "overlay ids must not contain path separators: {overlay_id}", + assert_eq!( + fs::read_link( + child_artifact + .join(".amber") + .join("control") + .join("router-control.sock") + ) + .expect("child artifact control alias should exist"), + child_current, + "child artifact control alias should point at the child current alias", ); } @@ -1560,15 +1921,16 @@ mod direct_input_tests { run_id: "run-123".to_string(), site_id: "direct_local".to_string(), kind: SiteKind::Direct, + direct_input_overlay_providers: BTreeSet::new(), children: BTreeMap::from([( 7, SiteControllerRuntimeChildRecord { child_id: 7, artifact_root: child_root.join("artifact").display().to_string(), assigned_components: Vec::new(), + controller_routes: Vec::new(), proxy_exports: BTreeMap::new(), direct_inputs: Vec::new(), - routed_inputs: Vec::new(), process_pid: None, published: true, }, diff --git a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs index 159a37d8..064e6f19 100644 --- a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs +++ b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs @@ -6,9 +6,7 @@ use std::{ path::{Path, PathBuf}, }; -use base64::Engine as _; use serde_json::json; -use tar::Builder; use super::*; @@ -17,8 +15,13 @@ const KUBERNETES_ROUTER_MESH_PORT: u16 = 24000; const KUBERNETES_ROUTER_CONTROL_PORT: u16 = 24100; const KUBERNETES_ROUTER_SERVICE_PATH: &str = "04-services/amber-router.yaml"; const KUBERNETES_ROUTER_NETPOL_PATH: &str = "05-networkpolicies/amber-router-netpol.yaml"; +const KUBERNETES_CONTROLLER_NETPOL_PATH: &str = + "05-networkpolicies/amber-site-controller-netpol.yaml"; const KUBERNETES_CONTROLLER_SEED_CONFIGMAP_PATH: &str = "01-configmaps/amber-site-controller-seed.yaml"; +const KUBERNETES_CONTROLLER_SEED_CONFIGMAP_PATH_PREFIX: &str = + "01-configmaps/amber-site-controller-seed"; +const KUBERNETES_CONTROLLER_SEED_CONFIGMAP_DATA_LIMIT: usize = 64 * 1024; const KUBERNETES_CONTROLLER_SERVICE_ACCOUNT_PATH: &str = "02-rbac/amber-site-controller-sa.yaml"; const KUBERNETES_CONTROLLER_ROLE_PATH: &str = "02-rbac/amber-site-controller-role.yaml"; const KUBERNETES_CONTROLLER_ROLE_BINDING_PATH: &str = @@ -36,20 +39,27 @@ const KUBERNETES_CONTROLLER_STATE_PATH: &str = "/amber/site/state/site-controlle const KUBERNETES_CONTROLLER_DESIRED_LINKS_PATH: &str = "/amber/site/state/desired-links.json"; const KUBERNETES_CONTROLLER_SEED_ROOT: &str = "/amber/seed"; +#[derive(Clone, Debug)] +struct KubernetesControllerSeedConfigMap { + path: String, + name: String, + items: Vec, + document: serde_json::Value, +} + pub fn inject_kubernetes_site_controller( artifact_root: &Path, plan: &SiteControllerPlan, controller_image: &str, ) -> Result<()> { let embedded_plan = build_embedded_kubernetes_controller_plan(plan); - let seed_configmap = - build_kubernetes_controller_seed_configmap(artifact_root, plan, &embedded_plan)?; + let seed_configmaps = + build_kubernetes_controller_seed_configmaps(artifact_root, plan, &embedded_plan)?; let labels = kubernetes_controller_labels(); - write_yaml_artifact( - artifact_root.join(KUBERNETES_CONTROLLER_SEED_CONFIGMAP_PATH), - &seed_configmap, - )?; + for configmap in &seed_configmaps { + write_yaml_artifact(artifact_root.join(&configmap.path), &configmap.document)?; + } write_yaml_artifact( artifact_root.join(KUBERNETES_CONTROLLER_SERVICE_ACCOUNT_PATH), &json!({ @@ -129,20 +139,24 @@ pub fn inject_kubernetes_site_controller( artifact_root.join(KUBERNETES_CONTROLLER_DEPLOYMENT_PATH), plan, controller_image, + &seed_configmaps, )?; - add_kubernetes_resource_paths( - artifact_root, - &[ - KUBERNETES_CONTROLLER_SEED_CONFIGMAP_PATH, - KUBERNETES_CONTROLLER_SERVICE_ACCOUNT_PATH, - KUBERNETES_CONTROLLER_ROLE_PATH, - KUBERNETES_CONTROLLER_ROLE_BINDING_PATH, - ], - )?; + let mut resource_paths = seed_configmaps + .iter() + .map(|configmap| configmap.path.as_str()) + .collect::>(); + resource_paths.extend([ + KUBERNETES_CONTROLLER_SERVICE_ACCOUNT_PATH, + KUBERNETES_CONTROLLER_ROLE_PATH, + KUBERNETES_CONTROLLER_ROLE_BINDING_PATH, + ]); + add_kubernetes_resource_paths(artifact_root, &resource_paths)?; + let peer_router_route_ports = peer_router_route_ports(&embedded_plan)?; ensure_kubernetes_router_allows_site_controller_ingress( artifact_root, - &peer_router_route_ports(&embedded_plan)?, - ) + &peer_router_route_ports, + )?; + ensure_kubernetes_site_controller_allows_router_egress(artifact_root, &peer_router_route_ports) } fn upsert_named_sequence_entry( @@ -219,6 +233,7 @@ fn patch_kubernetes_controller_deployment( path: PathBuf, plan: &SiteControllerPlan, controller_image: &str, + seed_configmaps: &[KubernetesControllerSeedConfigMap], ) -> Result<()> { let raw = fs::read_to_string(&path) .into_diagnostic() @@ -278,9 +293,10 @@ cp \ cp \ {KUBERNETES_CONTROLLER_SEED_ROOT}/desired-links.json \ {KUBERNETES_CONTROLLER_DESIRED_LINKS_PATH} -base64 -d \ - {KUBERNETES_CONTROLLER_SEED_ROOT}/artifact.tar.b64 | tar \ - -xf - -C {KUBERNETES_CONTROLLER_ARTIFACT_ROOT} +if [ -d {KUBERNETES_CONTROLLER_SEED_ROOT}/artifact ]; then + cp -R {KUBERNETES_CONTROLLER_SEED_ROOT}/artifact/. \ + {KUBERNETES_CONTROLLER_ARTIFACT_ROOT} +fi " ) ], @@ -334,13 +350,24 @@ base64 -d \ .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) .as_sequence_mut() .ok_or_else(|| miette::miette!("controller deployment volumes must be a sequence"))?; + let seed_sources = seed_configmaps + .iter() + .map(|configmap| { + json!({ + "configMap": { + "name": configmap.name.clone(), + "items": configmap.items.clone(), + } + }) + }) + .collect::>(); upsert_named_sequence_entry( volumes, KUBERNETES_CONTROLLER_SEED_VOLUME, serde_yaml::to_value(json!({ "name": KUBERNETES_CONTROLLER_SEED_VOLUME, - "configMap": { - "name": format!("{SITE_CONTROLLER_SERVICE_NAME}-seed"), + "projected": { + "sources": seed_sources, } })) .into_diagnostic() @@ -382,11 +409,42 @@ fn build_embedded_kubernetes_controller_plan(plan: &SiteControllerPlan) -> SiteC embedded } -fn build_kubernetes_controller_seed_configmap( +fn build_kubernetes_controller_seed_configmaps( + artifact_root: &Path, + plan: &SiteControllerPlan, + embedded_plan: &SiteControllerPlan, +) -> Result> { + let seed_files = build_kubernetes_controller_seed_files(artifact_root, plan, embedded_plan)?; + let mut chunks = Vec::>::new(); + let mut current = Vec::<(String, String)>::new(); + let mut current_size = 0usize; + for (path, contents) in seed_files { + let entry_size = path.len() + contents.len(); + if !current.is_empty() + && current_size + entry_size > KUBERNETES_CONTROLLER_SEED_CONFIGMAP_DATA_LIMIT + { + chunks.push(current); + current = Vec::new(); + current_size = 0; + } + current.push((path, contents)); + current_size += entry_size; + } + if !current.is_empty() { + chunks.push(current); + } + chunks + .into_iter() + .enumerate() + .map(|(index, files)| render_kubernetes_controller_seed_configmap(index, files)) + .collect() +} + +fn build_kubernetes_controller_seed_files( artifact_root: &Path, plan: &SiteControllerPlan, embedded_plan: &SiteControllerPlan, -) -> Result { +) -> Result> { let state_json = fs::read_to_string(&plan.state_path) .into_diagnostic() .wrap_err_with(|| format!("failed to read {}", plan.state_path))?; @@ -397,32 +455,67 @@ fn build_kubernetes_controller_seed_configmap( let embedded_plan_json = serde_json::to_string_pretty(embedded_plan) .into_diagnostic() .wrap_err("failed to serialize embedded kubernetes site controller plan")?; - Ok(json!({ - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": { - "name": format!("{SITE_CONTROLLER_SERVICE_NAME}-seed"), - "labels": kubernetes_controller_labels(), - }, - "data": { - "site-controller-plan.json": embedded_plan_json, - "site-controller-state.json": state_json, - "desired-links.json": desired_links_json, - "artifact.tar.b64": tar_directory_base64(artifact_root)?, - } - })) + let mut files = BTreeMap::from([ + ("site-controller-plan.json".to_string(), embedded_plan_json), + ("site-controller-state.json".to_string(), state_json), + ("desired-links.json".to_string(), desired_links_json), + ]); + for path in walk_files(artifact_root)? { + let relative = path + .strip_prefix(artifact_root) + .into_diagnostic() + .wrap_err_with(|| format!("failed to relativize {}", path.display()))?; + let relative = relative + .components() + .map(|component| component.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/"); + let contents = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + files.insert(format!("artifact/{relative}"), contents); + } + Ok(files.into_iter().collect()) } -fn tar_directory_base64(root: &Path) -> Result { - let mut tar = Builder::new(Vec::new()); - tar.append_dir_all(".", root) - .into_diagnostic() - .wrap_err_with(|| format!("failed to archive {}", root.display()))?; - let bytes = tar - .into_inner() - .into_diagnostic() - .wrap_err_with(|| format!("failed to finalize archive {}", root.display()))?; - Ok(base64::engine::general_purpose::STANDARD.encode(bytes)) +fn render_kubernetes_controller_seed_configmap( + index: usize, + files: Vec<(String, String)>, +) -> Result { + let name = if index == 0 { + format!("{SITE_CONTROLLER_SERVICE_NAME}-seed") + } else { + format!("{SITE_CONTROLLER_SERVICE_NAME}-seed-{index}") + }; + let path = if index == 0 { + KUBERNETES_CONTROLLER_SEED_CONFIGMAP_PATH.to_string() + } else { + format!("{KUBERNETES_CONTROLLER_SEED_CONFIGMAP_PATH_PREFIX}-{index}.yaml") + }; + let mut data = serde_json::Map::new(); + let mut items = Vec::with_capacity(files.len()); + for (entry_index, (relative_path, contents)) in files.into_iter().enumerate() { + let key = format!("seed-{entry_index:04}"); + data.insert(key.clone(), serde_json::Value::String(contents)); + items.push(json!({ + "key": key, + "path": relative_path, + })); + } + Ok(KubernetesControllerSeedConfigMap { + path, + name: name.clone(), + items, + document: json!({ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": name, + "labels": kubernetes_controller_labels(), + }, + "data": data, + }), + }) } fn add_kubernetes_resource_paths(artifact_root: &Path, paths: &[&str]) -> Result<()> { @@ -635,6 +728,111 @@ fn ensure_kubernetes_router_allows_site_controller_ingress( .wrap_err_with(|| format!("failed to write {}", path.display())) } +fn ensure_kubernetes_site_controller_allows_router_egress( + artifact_root: &Path, + route_ports: &BTreeSet, +) -> Result<()> { + if route_ports.is_empty() { + return Ok(()); + } + let path = artifact_root.join(KUBERNETES_CONTROLLER_NETPOL_PATH); + let raw = fs::read_to_string(&path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("invalid {}", path.display()))?; + let egress = document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("spec"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .and_then(|spec| spec.get_mut(yaml_string("egress"))) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!( + "site controller network policy {} is missing spec.egress", + path.display() + ) + })?; + let router_selector = BTreeMap::from([( + "amber.io/component".to_string(), + KUBERNETES_ROUTER_NAME.to_string(), + )]); + if let Some(rule) = egress.iter_mut().find(|rule| { + rule.as_mapping() + .and_then(|mapping| mapping.get(yaml_string("to"))) + .and_then(serde_yaml::Value::as_sequence) + .is_some_and(|to| { + to.iter().any(|peer| { + peer.as_mapping() + .and_then(|mapping| mapping.get(yaml_string("podSelector"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|selector_value| selector_value.get(yaml_string("matchLabels"))) + .and_then(serde_yaml::Value::as_mapping) + .is_some_and(|labels| { + labels.get(yaml_string("amber.io/component")) + == Some(&yaml_string(KUBERNETES_ROUTER_NAME)) + }) + }) + }) + }) { + let ports = rule + .as_mapping_mut() + .and_then(|mapping| mapping.get_mut(yaml_string("ports"))) + .and_then(serde_yaml::Value::as_sequence_mut) + .ok_or_else(|| { + miette::miette!( + "site controller network policy {} router egress rule is missing ports", + path.display() + ) + })?; + let existing_ports = ports + .iter() + .filter_map(|port| { + port.as_mapping() + .and_then(|entry| entry.get(yaml_string("port"))) + .and_then(serde_yaml::Value::as_u64) + .and_then(|port| u16::try_from(port).ok()) + }) + .collect::>(); + for port in route_ports { + if existing_ports.contains(port) { + continue; + } + ports.push( + serde_yaml::to_value(json!({ + "protocol": "TCP", + "port": port, + })) + .into_diagnostic() + .wrap_err("failed to serialize site controller router egress port")?, + ); + } + } else { + egress.push( + serde_yaml::to_value(json!({ + "to": [{ + "podSelector": { + "matchLabels": router_selector, + } + }], + "ports": route_ports.iter().map(|port| json!({ + "protocol": "TCP", + "port": port, + })).collect::>() + })) + .into_diagnostic() + .wrap_err("failed to serialize site controller router egress rule")?, + ); + } + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", path.display()))?; + fs::write(&path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", path.display())) +} + fn kubernetes_controller_labels() -> BTreeMap { BTreeMap::from([ ( @@ -676,7 +874,7 @@ fn write_yaml_artifact(path: PathBuf, value: &impl serde::Serialize) -> Result<( #[cfg(test)] mod tests { - use std::{collections::BTreeMap, net::SocketAddr}; + use std::{collections::BTreeMap, net::SocketAddr, path::PathBuf}; use amber_compiler::run_plan::SiteKind; @@ -722,17 +920,16 @@ mod tests { } } - #[test] - fn inject_kubernetes_site_controller_propagates_launch_env() { - let temp = tempfile::tempdir().expect("temp dir"); - let artifact_root = temp.path(); + fn write_kubernetes_controller_fixture(artifact_root: &Path) { fs::create_dir_all(artifact_root.join("04-services")).expect("services dir"); fs::create_dir_all(artifact_root.join("05-networkpolicies")).expect("netpol dir"); + fs::create_dir_all(artifact_root.join("03-deployments")).expect("deployments dir"); fs::write( artifact_root.join("kustomization.yaml"), "resources:\n - 03-deployments/amber-site-controller.yaml\n - \ 04-services/amber-site-controller.yaml\n - 04-services/amber-router.yaml\n - \ - 05-networkpolicies/amber-router-netpol.yaml\n", + 05-networkpolicies/amber-router-netpol.yaml\n - \ + 05-networkpolicies/amber-site-controller-netpol.yaml\n", ) .expect("kustomization should write"); fs::write( @@ -747,7 +944,27 @@ mod tests { amber-router-netpol\nspec:\n ingress: []\n", ) .expect("router netpol should write"); - fs::create_dir_all(artifact_root.join("03-deployments")).expect("deployments dir"); + fs::write( + artifact_root.join(KUBERNETES_CONTROLLER_NETPOL_PATH), + r#" +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: amber-site-controller-netpol +spec: + egress: + - to: + - podSelector: + matchLabels: + amber.io/component: amber-router + ports: + - protocol: TCP + port: 24000 + - protocol: TCP + port: 24100 +"#, + ) + .expect("site controller netpol should write"); fs::write( artifact_root.join(KUBERNETES_CONTROLLER_DEPLOYMENT_PATH), r#" @@ -788,8 +1005,9 @@ spec: "#, ) .expect("controller service should write"); + } - let plan = test_plan(artifact_root); + fn write_controller_state(plan: &SiteControllerPlan) { fs::write(&plan.state_path, "{}").expect("state should write"); fs::create_dir_all(&plan.site_state_root).expect("site state dir"); fs::write( @@ -797,6 +1015,35 @@ spec: "{}", ) .expect("desired links should write"); + } + + fn seed_configmap_paths(artifact_root: &Path) -> Vec { + let mut paths = + walk_files(&artifact_root.join("01-configmaps")).expect("seed configmaps should list"); + paths.sort(); + paths + } + + fn controller_pod_spec(deployment: &serde_yaml::Value) -> &serde_yaml::Mapping { + deployment + .as_mapping() + .and_then(|root| root.get(yaml_string("spec"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|spec| spec.get(yaml_string("template"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|template| template.get(yaml_string("spec"))) + .and_then(serde_yaml::Value::as_mapping) + .expect("controller deployment should include a pod spec") + } + + #[test] + fn inject_kubernetes_site_controller_propagates_launch_env() { + let temp = tempfile::tempdir().expect("temp dir"); + let artifact_root = temp.path(); + write_kubernetes_controller_fixture(artifact_root); + + let plan = test_plan(artifact_root); + write_controller_state(&plan); inject_kubernetes_site_controller( artifact_root, @@ -808,17 +1055,23 @@ spec: let deployment_raw = fs::read_to_string(artifact_root.join(KUBERNETES_CONTROLLER_DEPLOYMENT_PATH)) .expect("controller deployment should exist"); + assert!( + !deployment_raw.contains("artifact.tar.b64"), + "kubernetes controller bootstrap should seed ordinary files instead of a tarball" + ); + assert!( + !deployment_raw.contains("base64 -d"), + "kubernetes controller bootstrap should not decode a tarball at startup" + ); + assert!( + deployment_raw.contains("cp -R /amber/seed/artifact/."), + "kubernetes controller bootstrap should copy seeded artifact files directly" + ); let deployment: serde_yaml::Value = serde_yaml::from_str(&deployment_raw).expect("deployment yaml should parse"); - let containers = deployment - .as_mapping() - .and_then(|root| root.get(yaml_string("spec"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|spec| spec.get(yaml_string("template"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|template| template.get(yaml_string("spec"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|spec| spec.get(yaml_string("containers"))) + let pod_spec = controller_pod_spec(&deployment); + let containers = pod_spec + .get(yaml_string("containers")) .and_then(serde_yaml::Value::as_sequence) .expect("controller deployment should include containers"); assert_eq!( @@ -854,5 +1107,159 @@ spec: && mapping.get(yaml_string("value")) == Some(&yaml_string("kept")) }) })); + + let volumes = pod_spec + .get(yaml_string("volumes")) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller deployment should include volumes"); + let seed_volume = volumes + .iter() + .find(|volume| { + volume + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|name| name == KUBERNETES_CONTROLLER_SEED_VOLUME) + }) + .and_then(serde_yaml::Value::as_mapping) + .expect("controller deployment should include the seed volume"); + let projected_sources = seed_volume + .get(yaml_string("projected")) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|projected| projected.get(yaml_string("sources"))) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller seed volume should project configmaps"); + let projected_paths = projected_sources + .iter() + .flat_map(|source| { + source + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("configMap"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|config_map| config_map.get(yaml_string("items"))) + .and_then(serde_yaml::Value::as_sequence) + .into_iter() + .flatten() + .filter_map(|item| { + item.as_mapping() + .and_then(|mapping| mapping.get(yaml_string("path"))) + .and_then(serde_yaml::Value::as_str) + .map(str::to_string) + }) + .collect::>() + }) + .collect::>(); + assert!( + projected_paths + .iter() + .any(|path| path == "site-controller-plan.json") + ); + assert!( + projected_paths + .iter() + .any(|path| path == "site-controller-state.json") + ); + assert!( + projected_paths + .iter() + .any(|path| path == "desired-links.json") + ); + assert!( + projected_paths + .iter() + .any(|path| path == "artifact/kustomization.yaml") + ); + assert!( + seed_configmap_paths(artifact_root) + .iter() + .any(|path| path.file_name().and_then(|name| name.to_str()) + == Some("amber-site-controller-seed.yaml")), + "kustomization should still include the primary controller seed configmap" + ); + } + + #[test] + fn inject_kubernetes_site_controller_chunks_seed_configmaps_to_avoid_annotation_limits() { + let temp = tempfile::tempdir().expect("temp dir"); + let artifact_root = temp.path(); + write_kubernetes_controller_fixture(artifact_root); + fs::create_dir_all(artifact_root.join("seed-data")).expect("seed data dir"); + fs::write( + artifact_root.join("seed-data/large-a.txt"), + "a".repeat(KUBERNETES_CONTROLLER_SEED_CONFIGMAP_DATA_LIMIT / 2), + ) + .expect("large artifact A should write"); + fs::write( + artifact_root.join("seed-data/large-b.txt"), + "b".repeat(KUBERNETES_CONTROLLER_SEED_CONFIGMAP_DATA_LIMIT / 2), + ) + .expect("large artifact B should write"); + + let plan = test_plan(artifact_root); + write_controller_state(&plan); + + inject_kubernetes_site_controller( + artifact_root, + &plan, + "ghcr.io/rdi-foundation/amber-site-controller:test", + ) + .expect("kubernetes site controller injection should succeed"); + + let seed_paths = seed_configmap_paths(artifact_root); + assert!( + seed_paths.len() > 1, + "large controller seeds should be split across multiple configmaps" + ); + for seed_path in &seed_paths { + let raw = fs::read_to_string(seed_path).expect("seed configmap should read"); + let document: serde_yaml::Value = + serde_yaml::from_str(&raw).expect("seed configmap should parse"); + let data_size = document + .as_mapping() + .and_then(|root| root.get(yaml_string("data"))) + .and_then(serde_yaml::Value::as_mapping) + .expect("seed configmap should have data") + .iter() + .map(|(key, value)| { + key.as_str().unwrap_or_default().len() + + value.as_str().unwrap_or_default().len() + }) + .sum::(); + assert!( + data_size <= KUBERNETES_CONTROLLER_SEED_CONFIGMAP_DATA_LIMIT, + "seed configmap {} should stay under the chunk limit", + seed_path.display() + ); + } + + let deployment_raw = + fs::read_to_string(artifact_root.join(KUBERNETES_CONTROLLER_DEPLOYMENT_PATH)) + .expect("controller deployment should exist"); + let deployment: serde_yaml::Value = + serde_yaml::from_str(&deployment_raw).expect("deployment yaml should parse"); + let pod_spec = controller_pod_spec(&deployment); + let projected_sources = pod_spec + .get(yaml_string("volumes")) + .and_then(serde_yaml::Value::as_sequence) + .and_then(|volumes| { + volumes.iter().find(|volume| { + volume + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|name| name == KUBERNETES_CONTROLLER_SEED_VOLUME) + }) + }) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|mapping| mapping.get(yaml_string("projected"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|projected| projected.get(yaml_string("sources"))) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller deployment should project seed configmaps"); + assert_eq!( + projected_sources.len(), + seed_paths.len(), + "deployment should project every generated seed configmap" + ); } } diff --git a/runtime/site-controller/src/default_runtime/site_artifacts.rs b/runtime/site-controller/src/default_runtime/site_artifacts.rs index 437b6944..84ba179e 100644 --- a/runtime/site-controller/src/default_runtime/site_artifacts.rs +++ b/runtime/site-controller/src/default_runtime/site_artifacts.rs @@ -117,7 +117,6 @@ fn build_dynamic_compose_route_overlay_payload( assigned_components: &[String], component_mesh_dirs: &BTreeMap, proxy_exports: &BTreeMap, - routed_inputs: &[DynamicInputRouteRecord], existing_site_peer_identities: &BTreeMap, ) -> Result { let plan = read_embedded_compose_mesh_provision_plan(artifact_root)?; @@ -218,7 +217,6 @@ fn build_dynamic_compose_route_overlay_payload( }) .collect::>>()?; - let allowed_issuers = overlay_issuer_sets(routed_inputs)?; let mut payload = StoredRouteOverlayPayload { peers, inbound_routes: router_target.config.inbound, @@ -226,11 +224,8 @@ fn build_dynamic_compose_route_overlay_payload( augment_route_overlay_payload( &mut payload, proxy_exports, - routed_inputs, &component_peer_addrs, &peer_identities, - Some(&allowed_issuers), - false, )?; Ok(payload) } @@ -1311,23 +1306,12 @@ pub(super) fn prepare_dynamic_compose_child_artifact( component_mesh_dirs, } = build_dynamic_compose_mesh_plan(artifact_root, &runtime_spec.assigned_components)?; let mut mesh_plan = mesh_plan; - let router_mesh_port = router_mesh_port_from_plan( - &read_embedded_compose_mesh_provision_plan(artifact_root)?, - "compose", - )?; ensure_dynamic_proxy_export_component_routes( &mut mesh_plan, &runtime_spec.proxy_exports, &plan.router_identity_id, )?; rewrite_dynamic_direct_inputs(&mut mesh_plan, &runtime_spec.direct_inputs, live_components)?; - rewrite_dynamic_routed_inputs( - &mut mesh_plan, - &runtime_spec.routed_inputs, - SiteKind::Compose, - &plan.router_identity_id, - Some(router_mesh_port), - )?; let existing_peer_identities = required_existing_mesh_peer_identities(&mesh_plan, existing_site_peer_identities)?; mesh_plan.existing_peer_identities = existing_peer_identities.values().cloned().collect(); @@ -1349,7 +1333,6 @@ pub(super) fn prepare_dynamic_compose_child_artifact( &runtime_spec.assigned_components, &component_mesh_dirs, &runtime_spec.proxy_exports, - &runtime_spec.routed_inputs, existing_site_peer_identities, )?; write_json(&dynamic_route_overlay_path(artifact_root), &overlay_payload)?; @@ -1665,6 +1648,10 @@ fn inject_site_controller_peer_router_routes_into_plan( .find(|target| matches!(target.kind, MeshProvisionTargetKind::Router)) .ok_or_else(|| miette::miette!("mesh provision plan is missing a router target"))?; let inbound_route_id = site_controller_internal_route_id(local_site_id); + let local_target_peer_id = match &local_inbound_target { + InboundTarget::MeshForward { peer_id, .. } => Some(peer_id.clone()), + _ => None, + }; router .config .inbound @@ -1693,6 +1680,11 @@ fn inject_site_controller_peer_router_routes_into_plan( .cloned() .map(|identity| (identity.id.clone(), identity)) .collect::>(); + if let Some(peer_id) = local_target_peer_id + && !router.config.peers.iter().any(|peer| peer.id == peer_id) + { + router.config.peers.push(MeshPeerTemplate { id: peer_id }); + } for route in routes { existing_peer_identities.insert(route.peer_router.id.clone(), route.peer_router.clone()); if !router @@ -2026,7 +2018,6 @@ pub(super) fn prepare_dynamic_kubernetes_child_artifact( project_dynamic_child_mesh_scope(artifact_root, Some(&plan.mesh_scope))?; let plan_path = artifact_root.join("mesh-provision-plan.json"); let mesh_plan = read_embedded_kubernetes_mesh_provision_plan(artifact_root)?; - let router_mesh_port = router_mesh_port_from_plan(&mesh_plan, "kubernetes")?; let assigned = runtime_spec .assigned_components .iter() @@ -2086,13 +2077,6 @@ pub(super) fn prepare_dynamic_kubernetes_child_artifact( &runtime_spec.direct_inputs, live_components, )?; - rewrite_dynamic_routed_inputs( - &mut provision_plan, - &runtime_spec.routed_inputs, - SiteKind::Kubernetes, - &plan.router_identity_id, - Some(router_mesh_port), - )?; let provision_existing_peer_identities = required_existing_mesh_peer_identities(&provision_plan, existing_site_peer_identities)?; provision_plan.existing_peer_identities = provision_existing_peer_identities @@ -2184,7 +2168,6 @@ pub(super) fn write_kubernetes_live_route_overlay_payload( artifact_root: &Path, assigned_components: &[String], proxy_exports: &BTreeMap, - routed_inputs: &[DynamicInputRouteRecord], peer_identities: &BTreeMap, ) -> Result<()> { let provider_peer_addrs = kubernetes_peer_addrs_for_artifact(artifact_root)?; @@ -2197,11 +2180,8 @@ pub(super) fn write_kubernetes_live_route_overlay_payload( augment_route_overlay_payload( &mut payload, proxy_exports, - routed_inputs, &provider_peer_addrs, peer_identities, - None, - false, )?; write_dynamic_route_overlay_payload(artifact_root, &payload) } diff --git a/runtime/site-controller/src/lib.rs b/runtime/site-controller/src/lib.rs index 3ef9289f..ee9c188f 100644 --- a/runtime/site-controller/src/lib.rs +++ b/runtime/site-controller/src/lib.rs @@ -96,11 +96,11 @@ pub use self::{ site_controller_runtime_plan_from_controller_plan, site_state_path, }, state::{ - DynamicInputDirectRecord, DynamicInputRouteRecord, DynamicInputRouteTarget, - DynamicProxyExportRecord, SITE_CONTROLLER_INTERNAL_CAPABILITY, - SITE_CONTROLLER_SERVICE_NAME, SiteControllerPlan, authority_url_for_listen_addr, - build_site_controller_state, generate_control_state_auth_token, - site_controller_internal_route_id, write_control_state, write_site_controller_plan, + DynamicInputDirectRecord, DynamicInputRouteTarget, DynamicProxyExportRecord, + SITE_CONTROLLER_INTERNAL_CAPABILITY, SITE_CONTROLLER_SERVICE_NAME, SiteControllerPlan, + authority_url_for_listen_addr, build_site_controller_state, + generate_control_state_auth_token, site_controller_internal_route_id, write_control_state, + write_site_controller_plan, }, }; diff --git a/runtime/site-controller/src/orchestration.rs b/runtime/site-controller/src/orchestration.rs index eddd44c4..77e19772 100644 --- a/runtime/site-controller/src/orchestration.rs +++ b/runtime/site-controller/src/orchestration.rs @@ -99,6 +99,7 @@ pub(crate) struct ResolveExternalLinkUrlResponse { #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct PublishExternalSlotOverlayRequest { + pub(crate) site_id: String, pub(crate) overlay_id: String, pub(crate) slot_name: String, pub(crate) url: String, @@ -106,12 +107,14 @@ pub(crate) struct PublishExternalSlotOverlayRequest { #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct ClearExternalSlotOverlayRequest { + pub(crate) site_id: String, pub(crate) overlay_id: String, pub(crate) slot_name: String, } #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct PublishExportPeerOverlayRequest { + pub(crate) site_id: String, pub(crate) overlay_id: String, pub(crate) export_name: String, pub(crate) peer_id: String, @@ -123,6 +126,7 @@ pub(crate) struct PublishExportPeerOverlayRequest { #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct ClearExportPeerOverlayRequest { + pub(crate) site_id: String, pub(crate) overlay_id: String, pub(crate) export_name: String, pub(crate) peer_id: String, @@ -134,20 +138,34 @@ pub(crate) struct ClearExportPeerOverlayRequest { #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct RevokeDynamicCapabilityOriginOverlaysRequest { + pub(crate) site_id: String, pub(crate) overlay_ids: Vec, } #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct RemoteChildRuntimeRequest { + pub(crate) site_id: String, pub(crate) state: FrameworkControlState, pub(crate) child: LiveChildRecord, } #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct RemoteChildRollbackRequest { + pub(crate) site_id: String, pub(crate) child_id: u64, } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct RouterIdentityRequest { + pub(crate) site_id: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct RouterIdentityResponse { + pub(crate) id: String, + pub(crate) public_key_b64: String, +} + pub(super) fn control_state_step_error( step: &str, err: impl std::fmt::Display, @@ -314,69 +332,65 @@ pub(super) fn load_launched_site( }) } -async fn peer_router_identity_for_overlay( +fn site_router_control_endpoint( app: &ControlStateApp, site_id: &str, -) -> std::result::Result { - if let Some(identity) = app.controller_plan.peer_router_identities.get(site_id) { - return Ok(MeshPeer { - id: identity.id.clone(), - public_key: identity.public_key, - }); - } - let site_app = SiteControllerApp { - control: app.clone(), - ready: Arc::new(std::sync::atomic::AtomicBool::new(true)), - }; - let deadline = tokio::time::Instant::now() + Duration::from_secs(30); - let identity = loop { - match super::site_controller::peer_router_identity_via_router(&site_app, site_id).await { - Ok(identity) => break identity, - Err(err) - if err.0.code == ProtocolErrorCode::ControlStateUnavailable - && tokio::time::Instant::now() < deadline => - { - tokio::time::sleep(Duration::from_millis(250)).await; - } - Err(err) => return Err(err.0), - } - }; - Ok(MeshPeer { - id: identity.id, - public_key: identity.public_key, - }) -} - -fn local_router_control_endpoint( - app: &ControlStateApp, ) -> std::result::Result { - if let Some(raw) = app.controller_plan.local_router_control.as_deref() { + if site_id == app.controller_plan.site_id + && let Some(raw) = app.controller_plan.local_router_control.as_deref() + { return parse_control_endpoint(raw).map_err(|err| { protocol_error( ProtocolErrorCode::ControlStateUnavailable, - &format!("local router control endpoint is invalid: {err}"), + &format!("site `{site_id}` router control endpoint is invalid: {err}"), ) }); } - let state = load_site_manager_state(app, &app.controller_plan.site_id)?; + let state = load_site_manager_state(app, site_id)?; let raw = state.router_control.ok_or_else(|| { protocol_error( ProtocolErrorCode::ControlStateUnavailable, - "local router control endpoint is unavailable", + &format!("site `{site_id}` router control endpoint is unavailable"), ) })?; parse_control_endpoint(&raw).map_err(|err| { protocol_error( ProtocolErrorCode::ControlStateUnavailable, - &format!("local router control endpoint is invalid: {err}"), + &format!("site `{site_id}` router control endpoint is invalid: {err}"), ) }) } -pub(super) async fn local_router_identity_for_overlay( +pub(super) async fn router_identity_for_overlay( app: &ControlStateApp, + site_id: &str, ) -> std::result::Result { - let state = load_site_manager_state(app, &app.controller_plan.site_id)?; + let execution_site = { + let state = app.control_state.lock().await; + site_execution_site_from_state(&state, site_id)?.to_string() + }; + if execution_site != app.controller_plan.site_id { + let identity = remote_controller_post_json::<_, RouterIdentityResponse>( + app, + &execution_site, + "/v1/internal/router-identity", + &RouterIdentityRequest { + site_id: site_id.to_string(), + }, + ProtocolErrorCode::ControlStateUnavailable, + "read router identity", + ) + .await?; + return mesh_peer_from_router_identity_response(site_id, identity); + } + local_router_identity_for_overlay_site(app, site_id).await +} + +pub(super) async fn local_router_identity_for_overlay_site( + app: &ControlStateApp, + site_id: &str, +) -> std::result::Result { + let state = load_site_manager_state(app, site_id)?; if let (Some(id), Some(public_key_b64)) = (state.router_identity_id, state.router_public_key_b64) && let Ok(decoded) = @@ -385,20 +399,21 @@ pub(super) async fn local_router_identity_for_overlay( { return Ok(MeshPeer { id, public_key }); } - if let Ok(router_mesh) = app.runtime.load_live_site_router_mesh_config( - &site_controller_runtime_plan_from_controller_plan(&app.controller_plan), - ) { + if let Ok(router_mesh) = app + .runtime + .load_live_site_router_mesh_config(&site_runtime_plan_from_control_app(app, site_id)?) + { return Ok(MeshPeer { id: router_mesh.identity.id, public_key: router_mesh.identity.public_key, }); } - let identity = fetch_router_identity(&local_router_control_endpoint(app)?) + let identity = fetch_router_identity(&site_router_control_endpoint(app, site_id)?) .await .map_err(|err| { protocol_error( ProtocolErrorCode::ControlStateUnavailable, - &format!("failed to read local router identity: {err}"), + &format!("failed to read site `{site_id}` router identity: {err}"), ) })?; Ok(MeshPeer { @@ -407,8 +422,49 @@ pub(super) async fn local_router_identity_for_overlay( }) } -fn local_site_state_root(app: &ControlStateApp) -> &Path { - Path::new(&app.controller_plan.site_state_root) +fn mesh_peer_from_router_identity_response( + site_id: &str, + identity: RouterIdentityResponse, +) -> std::result::Result { + if let Ok(public_key) = + base64::engine::general_purpose::STANDARD.decode(identity.public_key_b64.as_bytes()) + && let Ok(public_key) = public_key.as_slice().try_into() + { + return Ok(MeshPeer { + id: identity.id, + public_key, + }); + } + Err(protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("site `{site_id}` router identity public key is invalid"), + )) +} + +fn site_state_root_for_site<'a>( + app: &'a ControlStateApp, + site_id: &str, +) -> std::borrow::Cow<'a, Path> { + if site_id == app.controller_plan.site_id { + return std::borrow::Cow::Borrowed(Path::new(&app.controller_plan.site_state_root)); + } + std::borrow::Cow::Owned(Path::new(&app.state_root).join(site_id)) +} + +fn site_runtime_plan_from_control_app( + app: &ControlStateApp, + site_id: &str, +) -> std::result::Result { + crate::default_runtime::runtime_plan_for_site_from_controller_plan( + &app.controller_plan, + site_id, + ) + .map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("site `{site_id}` runtime plan is unavailable: {err}"), + ) + }) } fn local_site_kind_from_state( @@ -428,24 +484,34 @@ fn local_site_kind_from_state( }) } +pub(super) fn site_execution_site_from_state<'a>( + state: &'a FrameworkControlState, + site_id: &'a str, +) -> std::result::Result<&'a str, ProtocolErrorResponse> { + let site = state.placement.offered_sites.get(site_id).ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("site `{site_id}` is missing from offered sites"), + ) + })?; + match site.kind { + SiteKind::Vm => site.controller_site.as_deref().ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("vm site `{site_id}` is missing its controller_site"), + ) + }), + SiteKind::Direct | SiteKind::Compose | SiteKind::Kubernetes => Ok(site_id), + } +} + pub(super) async fn resolve_external_link_url_local( app: &ControlStateApp, request: &ResolveExternalLinkUrlRequest, ) -> std::result::Result { - if request.link.provider_site != app.controller_plan.site_id { - return Err(controller_protocol_error( - ProtocolErrorCode::PublishFailed, - &request.link.provider_site, - "resolve provider link url", - format!( - "controller `{}` only resolves provider surfaces for its own site", - app.controller_plan.site_id - ), - )); - } - let provider = load_launched_site(app, &app.controller_plan.site_id)?; + let provider = load_launched_site(app, &request.link.provider_site)?; let provider_output_dir = provider_output_dir_for_request( - app, + site_state_root_for_site(app, &request.link.provider_site).as_ref(), request.child_id, Path::new(&provider.receipt.artifact_dir), request.provider_in_child, @@ -476,7 +542,7 @@ pub(super) async fn publish_external_slot_overlay_local( request: &PublishExternalSlotOverlayRequest, ) -> std::result::Result<(), ProtocolErrorResponse> { register_external_slot_with_retry( - &local_router_control_endpoint(app)?, + &site_router_control_endpoint(app, &request.site_id)?, &request.slot_name, &request.url, Duration::from_secs(30), @@ -485,14 +551,14 @@ pub(super) async fn publish_external_slot_overlay_local( .map_err(|err| { controller_protocol_error( ProtocolErrorCode::PublishFailed, - &app.controller_plan.site_id, + &request.site_id, "publish external slot overlay", err, ) })?; app.runtime .update_desired_overlay_for_consumer( - local_site_state_root(app), + site_state_root_for_site(app, &request.site_id).as_ref(), &request.overlay_id, DesiredExternalSlotOverlay { slot_name: request.slot_name.clone(), @@ -502,7 +568,7 @@ pub(super) async fn publish_external_slot_overlay_local( .map_err(|err| { controller_protocol_error( ProtocolErrorCode::ControlStateUnavailable, - &app.controller_plan.site_id, + &request.site_id, "persist desired external slot overlay", err, ) @@ -514,17 +580,20 @@ pub(super) async fn clear_external_slot_overlay_local( request: &ClearExternalSlotOverlayRequest, ) -> std::result::Result<(), ProtocolErrorResponse> { app.runtime - .clear_desired_overlay_for_consumer(local_site_state_root(app), &request.overlay_id) + .clear_desired_overlay_for_consumer( + site_state_root_for_site(app, &request.site_id).as_ref(), + &request.overlay_id, + ) .map_err(|err| { controller_protocol_error( ProtocolErrorCode::ControlStateUnavailable, - &app.controller_plan.site_id, + &request.site_id, "persist external slot overlay removal", err, ) })?; clear_external_slot_with_retry( - &local_router_control_endpoint(app)?, + &site_router_control_endpoint(app, &request.site_id)?, &request.slot_name, Duration::from_secs(30), ) @@ -532,7 +601,7 @@ pub(super) async fn clear_external_slot_overlay_local( .map_err(|err| { controller_protocol_error( ProtocolErrorCode::ControlStateUnavailable, - &app.controller_plan.site_id, + &request.site_id, "retract external slot overlay", err, ) @@ -544,7 +613,7 @@ pub(super) async fn publish_export_peer_overlay_local( request: &PublishExportPeerOverlayRequest, ) -> std::result::Result<(), ProtocolErrorResponse> { register_export_peer_with_retry( - &local_router_control_endpoint(app)?, + &site_router_control_endpoint(app, &request.site_id)?, &request.export_name, &request.peer_id, &request.peer_key_b64, @@ -556,14 +625,14 @@ pub(super) async fn publish_export_peer_overlay_local( .map_err(|err| { controller_protocol_error( ProtocolErrorCode::PublishFailed, - &app.controller_plan.site_id, + &request.site_id, "publish export-peer overlay", err, ) })?; app.runtime .update_desired_overlay_for_provider( - local_site_state_root(app), + site_state_root_for_site(app, &request.site_id).as_ref(), &request.overlay_id, DesiredExportPeerOverlay { export_name: request.export_name.clone(), @@ -576,7 +645,7 @@ pub(super) async fn publish_export_peer_overlay_local( .map_err(|err| { controller_protocol_error( ProtocolErrorCode::ControlStateUnavailable, - &app.controller_plan.site_id, + &request.site_id, "persist desired export-peer overlay", err, ) @@ -588,17 +657,20 @@ pub(super) async fn clear_export_peer_overlay_local( request: &ClearExportPeerOverlayRequest, ) -> std::result::Result<(), ProtocolErrorResponse> { app.runtime - .clear_desired_overlay_for_provider(local_site_state_root(app), &request.overlay_id) + .clear_desired_overlay_for_provider( + site_state_root_for_site(app, &request.site_id).as_ref(), + &request.overlay_id, + ) .map_err(|err| { controller_protocol_error( ProtocolErrorCode::ControlStateUnavailable, - &app.controller_plan.site_id, + &request.site_id, "persist export-peer overlay removal", err, ) })?; unregister_export_peer_with_retry( - &local_router_control_endpoint(app)?, + &site_router_control_endpoint(app, &request.site_id)?, &request.export_name, &request.peer_id, &request.peer_key_b64, @@ -610,7 +682,7 @@ pub(super) async fn clear_export_peer_overlay_local( .map_err(|err| { controller_protocol_error( ProtocolErrorCode::ControlStateUnavailable, - &app.controller_plan.site_id, + &request.site_id, "retract export-peer overlay", err, ) @@ -621,14 +693,14 @@ pub(super) async fn revoke_dynamic_capability_origin_overlays_local( app: &ControlStateApp, request: &RevokeDynamicCapabilityOriginOverlaysRequest, ) -> std::result::Result<(), ProtocolErrorResponse> { - let endpoint = local_router_control_endpoint(app)?; + let endpoint = site_router_control_endpoint(app, &request.site_id)?; for overlay_id in &request.overlay_ids { revoke_route_overlay_with_retry(&endpoint, overlay_id, Duration::from_secs(30)) .await .map_err(|err| { controller_protocol_error( ProtocolErrorCode::ControlStateUnavailable, - &app.controller_plan.site_id, + &request.site_id, "retract dynamic capability origin overlays", err, ) @@ -637,7 +709,7 @@ pub(super) async fn revoke_dynamic_capability_origin_overlays_local( Ok(()) } -fn peer_site_router_url( +fn remote_controller_base_url( app: &ControlStateApp, site_id: &str, ) -> std::result::Result { @@ -652,17 +724,17 @@ fn peer_site_router_url( protocol_error( ProtocolErrorCode::OriginUnavailable, &format!( - "site controller `{}` has no router forward path to peer site `{site_id}`", + "site controller `{}` has no router-local controller route to site `{site_id}`", app.controller_plan.site_id ), ) }) } -const PEER_CONTROLLER_REQUEST_RETRY_TIMEOUT: Duration = Duration::from_secs(30); -const PEER_CONTROLLER_REQUEST_RETRY_DELAY: Duration = Duration::from_millis(250); +const REMOTE_CONTROLLER_REQUEST_RETRY_TIMEOUT: Duration = Duration::from_secs(30); +const REMOTE_CONTROLLER_REQUEST_RETRY_DELAY: Duration = Duration::from_millis(250); -fn should_retry_peer_controller_status(status: StatusCode) -> bool { +fn should_retry_remote_controller_status(status: StatusCode) -> bool { matches!( status, StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE | StatusCode::GATEWAY_TIMEOUT @@ -674,31 +746,25 @@ async fn post_json_with_retry( url: &str, body: &TReq, ) -> std::result::Result { - let deadline = tokio::time::Instant::now() + PEER_CONTROLLER_REQUEST_RETRY_TIMEOUT; + let deadline = tokio::time::Instant::now() + REMOTE_CONTROLLER_REQUEST_RETRY_TIMEOUT; loop { - match client - .post(url) - .header(super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER, "1") - .json(body) - .send() - .await - { + match client.post(url).json(body).send().await { Ok(response) - if should_retry_peer_controller_status(response.status()) + if should_retry_remote_controller_status(response.status()) && tokio::time::Instant::now() < deadline => { - tokio::time::sleep(PEER_CONTROLLER_REQUEST_RETRY_DELAY).await; + tokio::time::sleep(REMOTE_CONTROLLER_REQUEST_RETRY_DELAY).await; } Ok(response) => return Ok(response), - Err(err) if tokio::time::Instant::now() < deadline => { - tokio::time::sleep(PEER_CONTROLLER_REQUEST_RETRY_DELAY).await; + Err(_) if tokio::time::Instant::now() < deadline => { + tokio::time::sleep(REMOTE_CONTROLLER_REQUEST_RETRY_DELAY).await; } Err(err) => return Err(err), } } } -async fn peer_controller_post_json_via_router Deserialize<'de>>( +async fn remote_controller_post_json Deserialize<'de>>( app: &ControlStateApp, site_id: &str, path: &str, @@ -710,7 +776,7 @@ async fn peer_controller_post_json_via_router D &app.client, &format!( "{}{}", - peer_site_router_url(app, site_id)?.trim_end_matches('/'), + remote_controller_base_url(app, site_id)?.trim_end_matches('/'), path ), body, @@ -735,11 +801,11 @@ async fn peer_controller_post_json_via_router D code, site_id, action, - format!("peer controller returned {status}"), + format!("remote controller returned {status}"), )) } -async fn peer_controller_post_no_content_via_router( +async fn remote_controller_post_no_content( app: &ControlStateApp, site_id: &str, path: &str, @@ -751,7 +817,7 @@ async fn peer_controller_post_no_content_via_router( &app.client, &format!( "{}{}", - peer_site_router_url(app, site_id)?.trim_end_matches('/'), + remote_controller_base_url(app, site_id)?.trim_end_matches('/'), path ), body, @@ -773,7 +839,7 @@ async fn peer_controller_post_no_content_via_router( code, site_id, action, - format!("peer controller returned {status}"), + format!("remote controller returned {status}"), )) } @@ -831,57 +897,15 @@ pub(super) async fn publish_dynamic_capability_origin( .await .map_err(|err| err.0); } - let url = format!( - "{}/v1/internal/dynamic-caps/origins/publish", - peer_site_router_url(app, site_id)?.trim_end_matches('/') - ); - let response = app - .client - .post(url) - .header(super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER, "1") - .json(request) - .send() - .await - .map_err(|err| { - protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "failed to reach site controller on site `{site_id}` through the site router \ - while publishing dynamic capability origin: {err}" - ), - ) - })?; - if response.status().is_success() { - return response.json().await.map_err(|err| { - protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "site controller on site `{site_id}` returned invalid JSON through the site \ - router while publishing dynamic capability origin: {err}" - ), - ) - }); - } - let status = response.status(); - let body = response.bytes().await.map_err(|err| { - protocol_error( - ProtocolErrorCode::OriginUnavailable, - &format!( - "failed to read site controller error response on site `{site_id}` through the \ - site router while publishing dynamic capability origin: {err}" - ), - ) - })?; - if let Ok(protocol_error) = serde_json::from_slice::(&body) { - return Err(protocol_error); - } - Err(protocol_error( + remote_controller_post_json( + app, + site_id, + "/v1/internal/dynamic-caps/origins/publish", + request, ProtocolErrorCode::OriginUnavailable, - &format!( - "site controller on site `{site_id}` returned {status} through the site router while \ - publishing dynamic capability origin" - ), - )) + "publish dynamic capability origin", + ) + .await } pub(super) fn dynamic_capability_origin_route_surface( @@ -1226,12 +1250,14 @@ pub(super) async fn prepare_child_on_site( child: &LiveChildRecord, site_id: &str, ) -> std::result::Result<(), ProtocolErrorResponse> { - if site_id != app.controller_plan.site_id { - return peer_controller_post_no_content_via_router( + let execution_site = site_execution_site_from_state(state, site_id)?; + if execution_site != app.controller_plan.site_id { + return remote_controller_post_no_content( app, - site_id, + execution_site, "/v1/internal/children/prepare", &RemoteChildRuntimeRequest { + site_id: site_id.to_string(), state: state.clone(), child: child.clone(), }, @@ -1250,7 +1276,7 @@ pub(super) async fn prepare_child_on_local_site( site_id: &str, ) -> std::result::Result<(), ProtocolErrorResponse> { app.runtime - .prepare_child(&app.controller_plan, state.clone(), child.clone()) + .prepare_child(&app.controller_plan, state.clone(), child.clone(), site_id) .await .map_err(|err| { controller_protocol_error( @@ -1268,12 +1294,14 @@ pub(super) async fn publish_child_on_site( child: &LiveChildRecord, site_id: &str, ) -> std::result::Result<(), ProtocolErrorResponse> { - if site_id != app.controller_plan.site_id { - return peer_controller_post_no_content_via_router( + let execution_site = site_execution_site_from_state(state, site_id)?; + if execution_site != app.controller_plan.site_id { + return remote_controller_post_no_content( app, - site_id, + execution_site, "/v1/internal/children/publish", &RemoteChildRuntimeRequest { + site_id: site_id.to_string(), state: state.clone(), child: child.clone(), }, @@ -1292,7 +1320,7 @@ pub(super) async fn publish_child_on_local_site( site_id: &str, ) -> std::result::Result<(), ProtocolErrorResponse> { app.runtime - .publish_child(&app.controller_plan, state.clone(), child.clone()) + .publish_child(&app.controller_plan, state.clone(), child.clone(), site_id) .await .map_err(|err| { controller_protocol_error( @@ -1332,12 +1360,18 @@ pub(super) async fn rollback_child_on_site( child_id: u64, site_id: &str, ) -> Result<()> { - if site_id != app.controller_plan.site_id { - return peer_controller_post_no_content_via_router( + let state = app.control_state.lock().await.clone(); + let execution_site = site_execution_site_from_state(&state, site_id) + .map_err(|err| miette::miette!(err.message))?; + if execution_site != app.controller_plan.site_id { + return remote_controller_post_no_content( app, - site_id, + execution_site, "/v1/internal/children/rollback", - &RemoteChildRollbackRequest { child_id }, + &RemoteChildRollbackRequest { + site_id: site_id.to_string(), + child_id, + }, ProtocolErrorCode::ControlStateUnavailable, "rollback child", ) @@ -1345,7 +1379,7 @@ pub(super) async fn rollback_child_on_site( .map_err(|err| miette::miette!(err.message)); } app.runtime - .rollback_child(&app.controller_plan, child_id) + .rollback_child(&app.controller_plan, child_id, site_id) .await } @@ -1355,12 +1389,14 @@ pub(super) async fn destroy_child_on_site( child: &LiveChildRecord, site_id: &str, ) -> std::result::Result<(), ProtocolErrorResponse> { - if site_id != app.controller_plan.site_id { - return peer_controller_post_no_content_via_router( + let execution_site = site_execution_site_from_state(state, site_id)?; + if execution_site != app.controller_plan.site_id { + return remote_controller_post_no_content( app, - site_id, + execution_site, "/v1/internal/children/destroy", &RemoteChildRuntimeRequest { + site_id: site_id.to_string(), state: state.clone(), child: child.clone(), }, @@ -1379,7 +1415,7 @@ pub(super) async fn destroy_child_on_local_site( site_id: &str, ) -> std::result::Result<(), ProtocolErrorResponse> { app.runtime - .destroy_child(&app.controller_plan, state.clone(), child.clone()) + .destroy_child(&app.controller_plan, state.clone(), child.clone(), site_id) .await .map_err(|err| { controller_protocol_error( @@ -1432,9 +1468,13 @@ pub(super) async fn publish_external_slot_overlay( let overlay_id = overlay_id_for_link_action(child, link, |action| { matches!(action, DynamicOverlayAction::ExternalSlot { .. }) })?; - let consumer_kind = { + let (consumer_kind, provider_execution_site, consumer_execution_site) = { let state = app.control_state.lock().await; - local_site_kind_from_state(&state, &link.consumer_site)? + ( + local_site_kind_from_state(&state, &link.consumer_site)?, + site_execution_site_from_state(&state, &link.provider_site)?.to_string(), + site_execution_site_from_state(&state, &link.consumer_site)?.to_string(), + ) }; let resolve_request = ResolveExternalLinkUrlRequest { child_id: child.child_id, @@ -1442,14 +1482,14 @@ pub(super) async fn publish_external_slot_overlay( consumer_kind, provider_in_child: provider_in_child_for_link(child, link), }; - let external_url = if link.provider_site == app.controller_plan.site_id { + let external_url = if provider_execution_site == app.controller_plan.site_id { resolve_external_link_url_local(app, &resolve_request) .await? .external_url } else { - peer_controller_post_json_via_router::<_, ResolveExternalLinkUrlResponse>( + remote_controller_post_json::<_, ResolveExternalLinkUrlResponse>( app, - &link.provider_site, + &provider_execution_site, "/v1/internal/link-overlays/external-url", &resolve_request, ProtocolErrorCode::PublishFailed, @@ -1459,16 +1499,17 @@ pub(super) async fn publish_external_slot_overlay( .external_url }; let publish_request = PublishExternalSlotOverlayRequest { + site_id: link.consumer_site.clone(), overlay_id: overlay_id.to_string(), slot_name: link.external_slot_name.clone(), url: external_url, }; - if link.consumer_site == app.controller_plan.site_id { + if consumer_execution_site == app.controller_plan.site_id { publish_external_slot_overlay_local(app, &publish_request).await } else { - peer_controller_post_no_content_via_router( + remote_controller_post_no_content( app, - &link.consumer_site, + &consumer_execution_site, "/v1/internal/link-overlays/external-slot/publish", &publish_request, ProtocolErrorCode::PublishFailed, @@ -1483,16 +1524,20 @@ pub(super) async fn publish_export_peer_overlay( child: &LiveChildRecord, link: &RunLink, ) -> std::result::Result<(), ProtocolErrorResponse> { + if link.provider_site == link.consumer_site { + return Ok(()); + } let overlay_id = overlay_id_for_link_action(child, link, |action| { matches!(action, DynamicOverlayAction::ExportPeer { .. }) })?; let route_id = export_peer_route_id(child, link)?; - let consumer_identity = if link.consumer_site == app.controller_plan.site_id { - local_router_identity_for_overlay(app).await? - } else { - peer_router_identity_for_overlay(app, &link.consumer_site).await? + let provider_execution_site = { + let state = app.control_state.lock().await; + site_execution_site_from_state(&state, &link.provider_site)?.to_string() }; + let consumer_identity = router_identity_for_overlay(app, &link.consumer_site).await?; let publish_request = PublishExportPeerOverlayRequest { + site_id: link.provider_site.clone(), overlay_id: overlay_id.to_string(), export_name: link.export_name.clone(), peer_id: consumer_identity.id, @@ -1501,12 +1546,12 @@ pub(super) async fn publish_export_peer_overlay( protocol: link.protocol.to_string(), route_id: Some(route_id), }; - if link.provider_site == app.controller_plan.site_id { + if provider_execution_site == app.controller_plan.site_id { publish_export_peer_overlay_local(app, &publish_request).await } else { - peer_controller_post_no_content_via_router( + remote_controller_post_no_content( app, - &link.provider_site, + &provider_execution_site, "/v1/internal/link-overlays/export-peer/publish", &publish_request, ProtocolErrorCode::PublishFailed, @@ -1561,7 +1606,7 @@ pub(super) fn link_still_required( } pub(super) fn provider_output_dir_for_request( - app: &ControlStateApp, + site_state_root: &Path, child_id: u64, provider_artifact_dir: &Path, provider_in_child: bool, @@ -1569,8 +1614,7 @@ pub(super) fn provider_output_dir_for_request( if !provider_in_child { return provider_artifact_dir.to_path_buf(); } - site_controller_runtime_child_root_for_site(local_site_state_root(app), child_id) - .join("artifact") + site_controller_runtime_child_root_for_site(site_state_root, child_id).join("artifact") } pub(super) fn provider_in_child_for_link(child: &LiveChildRecord, link: &RunLink) -> bool { @@ -1671,16 +1715,21 @@ pub(super) async fn clear_external_slot_overlay( return Ok(()); } } + let consumer_execution_site = { + let state = app.control_state.lock().await; + site_execution_site_from_state(&state, &link.consumer_site)?.to_string() + }; let clear_request = ClearExternalSlotOverlayRequest { + site_id: link.consumer_site.clone(), overlay_id: overlay_id.to_string(), slot_name: link.external_slot_name.clone(), }; - if link.consumer_site == app.controller_plan.site_id { + if consumer_execution_site == app.controller_plan.site_id { clear_external_slot_overlay_local(app, &clear_request).await } else { - peer_controller_post_no_content_via_router( + remote_controller_post_no_content( app, - &link.consumer_site, + &consumer_execution_site, "/v1/internal/link-overlays/external-slot/clear", &clear_request, ProtocolErrorCode::ControlStateUnavailable, @@ -1696,6 +1745,9 @@ pub(super) async fn clear_export_peer_overlay( child: &LiveChildRecord, link: &RunLink, ) -> std::result::Result<(), ProtocolErrorResponse> { + if link.provider_site == link.consumer_site { + return Ok(()); + } let overlay_id = overlay_id_for_link_action(child, link, |action| { matches!(action, DynamicOverlayAction::ExportPeer { .. }) })?; @@ -1706,12 +1758,13 @@ pub(super) async fn clear_export_peer_overlay( } } let route_id = export_peer_route_id(child, link)?; - let consumer_identity = if link.consumer_site == app.controller_plan.site_id { - local_router_identity_for_overlay(app).await? - } else { - peer_router_identity_for_overlay(app, &link.consumer_site).await? + let provider_execution_site = { + let state = app.control_state.lock().await; + site_execution_site_from_state(&state, &link.provider_site)?.to_string() }; + let consumer_identity = router_identity_for_overlay(app, &link.consumer_site).await?; let clear_request = ClearExportPeerOverlayRequest { + site_id: link.provider_site.clone(), overlay_id: overlay_id.to_string(), export_name: link.export_name.clone(), peer_id: consumer_identity.id, @@ -1720,12 +1773,12 @@ pub(super) async fn clear_export_peer_overlay( protocol: link.protocol.to_string(), route_id: Some(route_id), }; - if link.provider_site == app.controller_plan.site_id { + if provider_execution_site == app.controller_plan.site_id { clear_export_peer_overlay_local(app, &clear_request).await } else { - peer_controller_post_no_content_via_router( + remote_controller_post_no_content( app, - &link.provider_site, + &provider_execution_site, "/v1/internal/link-overlays/export-peer/clear", &clear_request, ProtocolErrorCode::ControlStateUnavailable, @@ -1788,15 +1841,20 @@ pub(super) async fn retract_dynamic_capability_origin_overlays( }; for (site_id, overlay_ids) in overlays_by_site { + let execution_site = { + let state = app.control_state.lock().await; + site_execution_site_from_state(&state, &site_id)?.to_string() + }; let revoke_request = RevokeDynamicCapabilityOriginOverlaysRequest { + site_id: site_id.clone(), overlay_ids: overlay_ids.into_iter().collect(), }; - if site_id == app.controller_plan.site_id { + if execution_site == app.controller_plan.site_id { revoke_dynamic_capability_origin_overlays_local(app, &revoke_request).await?; } else { - peer_controller_post_no_content_via_router( + remote_controller_post_no_content( app, - &site_id, + &execution_site, "/v1/internal/dynamic-caps/origins/revoke", &revoke_request, ProtocolErrorCode::ControlStateUnavailable, diff --git a/runtime/site-controller/src/planner.rs b/runtime/site-controller/src/planner.rs index cc038b2e..e53fc623 100644 --- a/runtime/site-controller/src/planner.rs +++ b/runtime/site-controller/src/planner.rs @@ -7,9 +7,9 @@ pub(crate) struct LocalChildRuntimeSpec { pub(crate) assigned_components: Vec, pub(crate) child_artifact_files: BTreeMap, pub(crate) site_artifact_files: BTreeMap, + pub(crate) controller_routes: Vec, pub(crate) proxy_exports: BTreeMap, pub(crate) direct_inputs: Vec, - pub(crate) routed_inputs: Vec, } pub(super) async fn prepare_child_record( @@ -1305,10 +1305,184 @@ pub(super) fn remove_child_record( } struct LocalChildRuntimeSpecInputs<'a> { + state: &'a FrameworkControlState, outputs: &'a BTreeMap, overlays: &'a [DynamicOverlayRecord], direct_inputs: &'a [DynamicInputDirectRecord], - routed_inputs: &'a [DynamicInputRouteRecord], +} + +fn parse_mesh_provision_plan_artifact( + artifact_files: &BTreeMap, +) -> std::result::Result, ProtocolErrorResponse> { + if let Some(raw) = artifact_files.get("mesh-provision-plan.json") { + return serde_json::from_str(raw).map(Some).map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("mesh-provision-plan.json is not a valid mesh provision plan: {err}"), + ) + }); + } + + if let Some(raw) = artifact_files.get("compose.yaml") { + let document: serde_yaml::Value = serde_yaml::from_str(raw).map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("compose.yaml is not valid YAML: {err}"), + ) + })?; + let content = document + .as_mapping() + .and_then(|root| root.get(serde_yaml::Value::String("configs".to_string()))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|configs| { + configs.get(serde_yaml::Value::String( + "amber-mesh-provision-plan".to_string(), + )) + }) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|config| config.get(serde_yaml::Value::String("content".to_string()))) + .and_then(serde_yaml::Value::as_str) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + "compose.yaml is missing configs.amber-mesh-provision-plan.content", + ) + })?; + return serde_json::from_str(content).map(Some).map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("compose.yaml has an invalid embedded mesh provision plan: {err}"), + ) + }); + } + + if let Some(raw) = artifact_files.get("01-configmaps/amber-mesh-provision.yaml") { + let document: serde_yaml::Value = serde_yaml::from_str(raw).map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("01-configmaps/amber-mesh-provision.yaml is not valid YAML: {err}"), + ) + })?; + let content = document + .as_mapping() + .and_then(|root| root.get(serde_yaml::Value::String("data".to_string()))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|data| data.get(serde_yaml::Value::String("mesh-plan.json".to_string()))) + .and_then(serde_yaml::Value::as_str) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + "01-configmaps/amber-mesh-provision.yaml is missing data.mesh-plan.json", + ) + })?; + return serde_json::from_str(content).map(Some).map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "01-configmaps/amber-mesh-provision.yaml has an invalid embedded mesh \ + provision plan: {err}" + ), + ) + }); + } + + Ok(None) +} + +fn framework_controller_route_grants_for_child( + state: &FrameworkControlState, + site_id: &str, + child_moniker_set: &BTreeSet<&str>, +) -> BTreeMap> { + let mut grants = BTreeMap::>::new(); + for record in state.capability_instances.values() { + if record.controller_site_id != site_id { + continue; + } + if record.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME { + continue; + } + if !child_moniker_set.contains(record.recipient_component_moniker.as_str()) { + continue; + } + grants + .entry(record.route_id.clone()) + .or_default() + .insert(record.recipient_peer_id.clone()); + } + grants +} + +fn controller_routes_for_child( + state: &FrameworkControlState, + site_id: &str, + desired_site_plan: &amber_compiler::run_plan::RunSitePlan, + child_moniker_set: &BTreeSet<&str>, +) -> std::result::Result, ProtocolErrorResponse> { + let grants = framework_controller_route_grants_for_child(state, site_id, child_moniker_set); + let Some(mesh_plan) = parse_mesh_provision_plan_artifact(&desired_site_plan.artifact_files)? + else { + if grants.is_empty() { + return Ok(Vec::new()); + } + return Err(protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "site `{site_id}` has dynamic framework.component grants but no mesh provision \ + plan artifact" + ), + )); + }; + + let mut routes = Vec::new(); + for target in mesh_plan.targets { + if !matches!(target.kind, amber_mesh::MeshProvisionTargetKind::Component) { + continue; + } + if !target.config.inbound.iter().any(|route| { + route.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + }) { + continue; + } + for mut route in target.config.inbound { + if route.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + { + route + .allowed_issuers + .retain(|issuer| child_moniker_set.contains(issuer.as_str())); + if !route.allowed_issuers.is_empty() { + routes.push(route); + } + continue; + } + let Some(allowed_issuers) = grants.get(&route.route_id) else { + continue; + }; + route.allowed_issuers = allowed_issuers.iter().cloned().collect(); + routes.push(route); + } + } + + let planned_route_ids = routes + .iter() + .map(|route| route.route_id.as_str()) + .collect::>(); + let missing = grants + .keys() + .filter(|route_id| !planned_route_ids.contains(route_id.as_str())) + .cloned() + .collect::>(); + if !missing.is_empty() { + return Err(protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "site `{site_id}` mesh plan is missing dynamic framework.component controller \ + routes: {}", + missing.join(", ") + ), + )); + } + Ok(routes) } fn build_local_child_runtime_spec_from_site_plan( @@ -1448,6 +1622,12 @@ fn build_local_child_runtime_spec_from_site_plan( assigned_components, child_artifact_files, site_artifact_files: desired_site_plan.artifact_files.clone(), + controller_routes: controller_routes_for_child( + inputs.state, + site_id, + desired_site_plan, + &child_moniker_set, + )?, proxy_exports, direct_inputs: inputs .direct_inputs @@ -1460,17 +1640,6 @@ fn build_local_child_runtime_spec_from_site_plan( }) .cloned() .collect(), - routed_inputs: inputs - .routed_inputs - .iter() - .filter(|input| input.component == child_monikers[&fragment.root_component_id]) - .filter(|input| { - assignments - .get(input.component.as_str()) - .is_some_and(|assigned_site| assigned_site == site_id) - }) - .cloned() - .collect(), }) } @@ -1583,6 +1752,7 @@ pub(super) fn build_local_child_runtime_spec( &live_assignments, fragment, LocalChildRuntimeSpecInputs { + state, outputs: &child.outputs, overlays: &child.overlays, direct_inputs: &dynamic_input_direct_records( @@ -1590,11 +1760,6 @@ pub(super) fn build_local_child_runtime_spec( fragment, &child.input_bindings, ), - routed_inputs: &dynamic_input_route_records( - &live_assignments, - fragment, - &child.input_bindings, - ), }, ) } @@ -1653,60 +1818,6 @@ pub(super) fn child_input_binding_records( .collect() } -pub(super) fn dynamic_input_route_records( - assignments: &BTreeMap, - fragment: &LiveScenarioFragment, - input_bindings: &[ChildInputBindingRecord], -) -> Vec { - let Some(root_component) = fragment - .components - .iter() - .find(|component| component.id == fragment.root_component_id) - else { - return Vec::new(); - }; - let Some(child_site) = assignments.get(root_component.moniker.as_str()) else { - return Vec::new(); - }; - let fragment_components = fragment - .components - .iter() - .map(|component| component.moniker.as_str()) - .collect::>(); - input_bindings - .iter() - .filter_map(|binding| { - let [source] = binding.sources.as_slice() else { - return None; - }; - let BindingFromIr::Component { provide, .. } = &source.from else { - return None; - }; - let provider_component = source.component_moniker.as_ref()?; - if fragment_components.contains(provider_component.as_str()) { - return None; - } - let provider_site = assignments.get(provider_component.as_str())?; - let protocol = match binding.decl.kind.transport() { - CapabilityTransport::Http => "http", - CapabilityTransport::NonNetwork => return None, - _ => return None, - }; - (provider_site != child_site).then(|| DynamicInputRouteRecord { - component: root_component.moniker.clone(), - slot: binding.slot.clone(), - provider_component: provider_component.clone(), - protocol: protocol.to_string(), - capability_kind: binding.decl.kind.to_string(), - capability_profile: binding.decl.profile.clone(), - target: DynamicInputRouteTarget::ComponentProvide { - provide: provide.clone(), - }, - }) - }) - .collect() -} - pub(super) fn dynamic_input_direct_records( assignments: &BTreeMap, fragment: &LiveScenarioFragment, @@ -1874,6 +1985,54 @@ pub(super) fn create_child_response(child: &LiveChildRecord) -> CreateChildRespo } } +fn merge_framework_controller_component_ir( + existing: &mut ComponentIr, + fragment_controller: &ComponentIr, + id_map: &BTreeMap, +) -> std::result::Result<(), ProtocolErrorResponse> { + let mut fragment_metadata = amber_compiler::run_plan::framework_component_controller_metadata( + fragment_controller.metadata.as_ref(), + ) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + "dynamic child fragment controller is missing its framework metadata", + ) + })?; + let mut existing_metadata = amber_compiler::run_plan::framework_component_controller_metadata( + existing.metadata.as_ref(), + ) + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + "live framework.component controller is missing its framework metadata", + ) + })?; + for grant in fragment_metadata.grants.values_mut() { + if let Some(remapped) = id_map.get(&grant.authority_realm_id) { + grant.authority_realm_id = *remapped; + } + } + for (slot, decl) in &fragment_controller.slots { + existing + .slots + .entry(slot.clone()) + .or_insert_with(|| decl.clone()); + } + for (provide, decl) in &fragment_controller.provides { + existing + .provides + .entry(provide.clone()) + .or_insert_with(|| decl.clone()); + } + existing_metadata.grants.extend(fragment_metadata.grants); + existing.metadata = Some( + serde_json::to_value(existing_metadata) + .expect("framework controller metadata should serialize"), + ); + Ok(()) +} + pub(super) fn extract_live_child_fragment( state: &mut FrameworkControlState, compiled: &CompiledScenario, @@ -1899,20 +2058,73 @@ pub(super) fn extract_live_child_fragment( ) })? .0; + let existing_controllers_by_site = state + .base_scenario + .components + .iter() + .filter_map(|component| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .map(|metadata| (metadata.execution_site, component.clone())) + }) + .collect::>(); let mut id_map = BTreeMap::new(); + let mut reused_controller_component_ids = BTreeSet::new(); + let mut reused_controller_fragments = BTreeMap::>::new(); for component in &compiled.scenario_ir().components { if component.id == wrapper_root { continue; } + if let Some(metadata) = amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) && let Some(existing_controller) = + existing_controllers_by_site.get(&metadata.execution_site) + { + id_map.insert(component.id, existing_controller.id); + reused_controller_component_ids.insert(component.id); + reused_controller_fragments + .entry(existing_controller.id) + .or_default() + .push(component.clone()); + continue; + } id_map.insert(component.id, state.next_component_id); state.next_component_id += 1; } + let mut controller_replacements = BTreeMap::::new(); + for (existing_controller_id, fragments) in reused_controller_fragments { + let existing_controller = state + .base_scenario + .components + .iter() + .find(|component| component.id == existing_controller_id) + .cloned() + .ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!( + "live framework.component controller {} disappeared while preparing a \ + dynamic child fragment", + existing_controller_id + ), + ) + })?; + let replacement = controller_replacements + .entry(existing_controller_id) + .or_insert(existing_controller); + for fragment_controller in &fragments { + merge_framework_controller_component_ir(replacement, fragment_controller, &id_map)?; + } + } let mut components = compiled .scenario_ir() .components .iter() - .filter(|component| component.id != wrapper_root) + .filter(|component| { + component.id != wrapper_root && !reused_controller_component_ids.contains(&component.id) + }) .map(|component| { let mut component = component.clone(); component.id = *id_map @@ -1941,6 +2153,7 @@ pub(super) fn extract_live_child_fragment( component }) .collect::>(); + components.extend(controller_replacements.into_values()); components.sort_by(|left, right| left.id.cmp(&right.id)); let mut bindings = Vec::new(); @@ -2199,6 +2412,18 @@ pub(super) fn collect_capability_instances( Ok(records) } +fn framework_capability_recipient_peer_id( + recipient_component_moniker: &str, + recipient_site_id: &str, + controller_site_id: &str, +) -> String { + if recipient_site_id == controller_site_id { + recipient_component_moniker.to_string() + } else { + format!("/site/{controller_site_id}/router") + } +} + pub(super) fn collect_capability_instance_from_binding( records: &mut BTreeMap, binding: &BindingIr, @@ -2245,13 +2470,9 @@ pub(super) fn collect_capability_instance_from_binding( let Some(metadata) = controller_metadata_by_id.get(component) else { return Ok(()); }; - let Some(grant) = metadata.grants.get(provide) else { - return Ok(()); - }; let provider_moniker = moniker_by_id.get(component).cloned().ok_or_else(|| { miette::miette!( - "framework.component controller component id {component} is missing from \ - the authoritative live graph" + "framework.component controller component id {component} is missing from the authoritative live graph" ) })?; let route_id = amber_mesh::component_route_id( @@ -2259,27 +2480,66 @@ pub(super) fn collect_capability_instance_from_binding( provide, amber_mesh::MeshProtocol::Http, ); + if let Some(grant) = metadata.grants.get(provide) { + let cap_instance_id = framework_cap_instance_id( + grant.authority_realm_moniker.as_str(), + recipient_component_moniker.as_str(), + &binding.to.component.to_string(), + &binding.to.slot, + "component", + ); + records.insert( + cap_instance_id.clone(), + CapabilityInstanceRecord { + cap_instance_id, + route_id, + authority_realm_id: grant.authority_realm_id, + authority_realm_moniker: grant.authority_realm_moniker.clone(), + recipient_component_id: binding.to.component, + recipient_component_moniker: recipient_component_moniker.clone(), + recipient_peer_id: framework_capability_recipient_peer_id( + recipient_component_moniker.as_str(), + recipient_site_id.as_str(), + metadata.execution_site.as_str(), + ), + recipient_site_id, + controller_site_id: metadata.execution_site.clone(), + managed_site_id: grant.managed_site.clone(), + capability: "component".to_string(), + slot: binding.to.slot.clone(), + generation, + }, + ); + return Ok(()); + } + if provide != amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME { + return Ok(()); + } let cap_instance_id = framework_cap_instance_id( - grant.authority_realm_moniker.as_str(), + provider_moniker.as_str(), recipient_component_moniker.as_str(), &binding.to.component.to_string(), &binding.to.slot, - "component", + provide, ); records.insert( cap_instance_id.clone(), CapabilityInstanceRecord { cap_instance_id, route_id, - authority_realm_id: grant.authority_realm_id, - authority_realm_moniker: grant.authority_realm_moniker.clone(), + authority_realm_id: *component, + authority_realm_moniker: provider_moniker, recipient_component_id: binding.to.component, recipient_component_moniker: recipient_component_moniker.clone(), - recipient_peer_id: recipient_component_moniker, + recipient_peer_id: framework_capability_recipient_peer_id( + recipient_component_moniker.as_str(), + recipient_site_id.as_str(), + metadata.execution_site.as_str(), + ), recipient_site_id, controller_site_id: metadata.execution_site.clone(), - managed_site_id: grant.managed_site.clone(), - capability: "component".to_string(), + managed_site_id: String::new(), + capability: provide.clone(), slot: binding.to.slot.clone(), generation, }, diff --git a/runtime/site-controller/src/runtime_api.rs b/runtime/site-controller/src/runtime_api.rs index a035e4d7..0eae1d69 100644 --- a/runtime/site-controller/src/runtime_api.rs +++ b/runtime/site-controller/src/runtime_api.rs @@ -44,6 +44,7 @@ pub(crate) trait SiteControllerRuntime: Send + Sync { plan: &'a SiteControllerPlan, state: FrameworkControlState, child: LiveChildRecord, + site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()>; fn publish_child<'a>( @@ -51,12 +52,14 @@ pub(crate) trait SiteControllerRuntime: Send + Sync { plan: &'a SiteControllerPlan, state: FrameworkControlState, child: LiveChildRecord, + site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()>; fn rollback_child<'a>( &'a self, plan: &'a SiteControllerPlan, child_id: u64, + site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()>; fn destroy_child<'a>( @@ -64,6 +67,7 @@ pub(crate) trait SiteControllerRuntime: Send + Sync { plan: &'a SiteControllerPlan, state: FrameworkControlState, child: LiveChildRecord, + site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()>; fn collect_live_component_runtime_metadata( diff --git a/runtime/site-controller/src/site_controller.rs b/runtime/site-controller/src/site_controller.rs index 727724d3..5455c417 100644 --- a/runtime/site-controller/src/site_controller.rs +++ b/runtime/site-controller/src/site_controller.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use amber_mesh::{MeshIdentityPublic, dynamic_caps::HeldListResponse}; +use amber_mesh::dynamic_caps::HeldListResponse; use amber_proxy::{ControlEndpoint, fetch_router_identity}; use axum::{ Json, Router, @@ -33,13 +33,13 @@ use super::{ PublishExportPeerOverlayRequest, PublishExternalSlotOverlayRequest, RemoteChildRollbackRequest, RemoteChildRuntimeRequest, ResolveExternalLinkUrlRequest, ResolveExternalLinkUrlResponse, RevokeDynamicCapabilityOriginOverlaysRequest, - clear_export_peer_overlay_local, clear_external_slot_overlay_local, - destroy_child_on_local_site, execute_create_child, execute_destroy_child, - prepare_child_on_local_site, publish_child_on_local_site, - publish_dynamic_capability_origin_local, publish_export_peer_overlay_local, - publish_external_slot_overlay_local, recover_control_state, - resolve_external_link_url_local, revoke_dynamic_capability_origin_overlays_local, - rollback_child_on_site, + RouterIdentityRequest, RouterIdentityResponse, clear_export_peer_overlay_local, + clear_external_slot_overlay_local, destroy_child_on_local_site, execute_create_child, + execute_destroy_child, local_router_identity_for_overlay_site, prepare_child_on_local_site, + publish_child_on_local_site, publish_dynamic_capability_origin_local, + publish_export_peer_overlay_local, publish_external_slot_overlay_local, + recover_control_state, resolve_external_link_url_local, + revoke_dynamic_capability_origin_overlays_local, rollback_child_on_site, }, planner::{ ControlStateApp, LocalDynamicCapabilityOriginApp, SiteControllerApp, protocol_error, @@ -157,6 +157,10 @@ pub(crate) fn site_controller_router(app_state: SiteControllerApp) -> Router { "/v1/internal/link-overlays/external-url", post(resolve_external_link_url), ) + .route( + "/v1/internal/router-identity", + post(get_internal_router_identity_route), + ) .route( "/v1/internal/children/prepare", post(prepare_child_runtime_route), @@ -234,17 +238,18 @@ async fn get_site_controller_state_route( State(app): State, headers: HeaderMap, ) -> std::result::Result, ProtocolApiError> { - authorize_local_controller_request(&app, &headers)?; + authorize_local_controller_request(&app, &headers).await?; Ok(Json(app.control.control_state.lock().await.clone())) } -#[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct RouterIdentityResponse { - pub(crate) id: String, - pub(crate) public_key_b64: String, +fn controller_local_only(headers: &HeaderMap) -> bool { + headers + .get(CONTROLLER_LOCAL_ONLY_HEADER) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value == "1") } -pub(super) fn authorize_local_controller_request( +pub(super) async fn authorize_local_controller_request( app: &SiteControllerApp, headers: &HeaderMap, ) -> std::result::Result<(), ProtocolApiError> { @@ -254,7 +259,38 @@ pub(super) fn authorize_local_controller_request( )); } let route_id = required_header(headers, FRAMEWORK_ROUTE_ID_HEADER)?; - let expected_route_id = site_controller_internal_route_id(&app.control.controller_plan.site_id); + let expected_route_id = { + let state = app.control.control_state.lock().await; + let scenario = amber_scenario::Scenario::try_from(state.base_scenario.clone()).map_err( + |err| { + ProtocolApiError::control_state_unavailable(format!( + "failed to read lowered scenario while authorizing a local controller request: {err}" + )) + }, + )?; + let controller_peer_id = scenario + .components_iter() + .find(|(_, component)| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .is_some_and(|metadata| { + metadata.execution_site == app.control.controller_plan.site_id + }) + }) + .map(|(_, component)| component.moniker.as_str().to_string()) + .ok_or_else(|| { + ProtocolApiError::control_state_unavailable(format!( + "site controller component for site `{}` is missing from the lowered scenario", + app.control.controller_plan.site_id, + )) + })?; + amber_mesh::component_route_id( + controller_peer_id.as_str(), + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + amber_mesh::MeshProtocol::Http, + ) + }; if route_id != expected_route_id { return Err(ProtocolApiError::unauthorized(format!( "local controller endpoint requires internal route `{expected_route_id}`" @@ -312,7 +348,7 @@ async fn get_router_identity_route( State(app): State, headers: HeaderMap, ) -> std::result::Result, ProtocolApiError> { - authorize_local_controller_request(&app, &headers)?; + authorize_remote_controller_request(&app, &headers).await?; let endpoint = local_router_control_endpoint(&app)?; let identity = fetch_router_identity(&endpoint).await.map_err(|err| { ProtocolApiError::control_state_unavailable(format!( @@ -325,6 +361,29 @@ async fn get_router_identity_route( })) } +async fn get_internal_router_identity_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + authorize_remote_controller_request(&app, &headers).await?; + let execution_site = { + let state = app.control.control_state.lock().await; + super::orchestration::site_execution_site_from_state(&state, &request.site_id)?.to_string() + }; + if execution_site != app.control.controller_plan.site_id { + return Err(ProtocolApiError::control_state_unavailable(format!( + "site `{}` is controlled by `{execution_site}`, not `{}`", + request.site_id, app.control.controller_plan.site_id, + ))); + } + let identity = local_router_identity_for_overlay_site(&app.control, &request.site_id).await?; + Ok(Json(RouterIdentityResponse { + id: identity.id, + public_key_b64: base64::engine::general_purpose::STANDARD.encode(identity.public_key), + })) +} + pub(super) async fn authorize_public_request( app: &SiteControllerApp, headers: &HeaderMap, @@ -338,8 +397,38 @@ pub(super) async fn authorize_public_request( Ok((record, state)) } +pub(super) async fn authorize_remote_controller_request( + app: &SiteControllerApp, + headers: &HeaderMap, +) -> std::result::Result<(), ProtocolApiError> { + let (record, state) = authorize_public_request(app, headers).await?; + if record.capability != amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME { + return Err(ProtocolApiError::unauthorized( + "remote site controller requests must arrive over the ordinary controller internal \ + capability" + .to_string(), + )); + } + let is_controller = state + .base_scenario + .components + .iter() + .find(|component| component.id == record.recipient_component_id) + .is_some_and(|component| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .is_some() + }); + if !is_controller { + return Err(ProtocolApiError::unauthorized( + "remote site controller requests must target a synthetic controller component" + .to_string(), + )); + } + Ok(()) +} pub(super) fn authorize_dynamic_caps_sidecar_request( - _app: &SiteControllerApp, headers: &HeaderMap, expected_component_id: &str, ) -> std::result::Result<(), ProtocolApiError> { @@ -366,11 +455,27 @@ pub(super) fn authorize_dynamic_caps_sidecar_request( Ok(()) } -fn controller_local_only(headers: &HeaderMap) -> bool { - headers - .get(CONTROLLER_LOCAL_ONLY_HEADER) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value == "1") +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum DynamicCapsRequestAuth { + Sidecar, + RemoteController, +} + +pub(super) async fn authorize_dynamic_caps_request( + app: &SiteControllerApp, + headers: &HeaderMap, + expected_component_id: &str, +) -> std::result::Result { + let peer_id = required_header(headers, FRAMEWORK_PEER_ID_HEADER)?; + let expected_sidecar_peer = + dynamic_caps::moniker_from_logical_component_id(expected_component_id) + .map_err(ProtocolApiError::from)?; + if peer_id == expected_sidecar_peer { + authorize_dynamic_caps_sidecar_request(headers, expected_component_id)?; + return Ok(DynamicCapsRequestAuth::Sidecar); + } + authorize_remote_controller_request(app, headers).await?; + Ok(DynamicCapsRequestAuth::RemoteController) } fn ensure_controller_ready(app: &SiteControllerApp) -> std::result::Result<(), ProtocolApiError> { @@ -382,103 +487,56 @@ fn ensure_controller_ready(app: &SiteControllerApp) -> std::result::Result<(), P )) } -fn peer_site_router_url_for_site( - app: &SiteControllerApp, - site_id: &str, -) -> std::result::Result { +fn remote_controller_urls(app: &SiteControllerApp) -> BTreeMap { app.control .controller_plan .peer_site_router_urls - .get(site_id) - .cloned() - .ok_or_else(|| { - ProtocolApiError::control_state_unavailable(format!( - "site controller `{}` has no router forward path to peer site `{site_id}`", - app.control.controller_plan.site_id - )) - }) -} - -fn peer_site_ids(app: &SiteControllerApp) -> Vec { - app.control - .controller_plan - .peer_site_router_urls - .keys() - .cloned() + .iter() + .filter(|(site_id, _)| site_id.as_str() != app.control.controller_plan.site_id) + .map(|(site_id, url)| (site_id.clone(), url.clone())) .collect() } -async fn peer_dynamic_caps_post_via_router( +fn remote_controller_base_url( app: &SiteControllerApp, site_id: &str, - path: &str, - body: &TReq, -) -> std::result::Result { - let response = app - .control - .client - .post(format!( - "{}{}", - peer_site_router_url_for_site(app, site_id)?.trim_end_matches('/'), - path - )) - .header(CONTROLLER_LOCAL_ONLY_HEADER, "1") - .json(body) - .send() - .await - .map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "failed to reach site controller for site `{site_id}` through site router: {err}" - )) - })?; - if response.status().is_success() { - return response.json().await.map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "site controller for site `{site_id}` returned invalid JSON through site router: \ - {err}" - )) - }); - } - let status = response.status(); - let body = response.bytes().await.map_err(|err| { +) -> std::result::Result { + remote_controller_urls(app).remove(site_id).ok_or_else(|| { ProtocolApiError::control_state_unavailable(format!( - "failed to read site controller error response for site `{site_id}`: {err}" + "site controller `{}` has no router-local controller route to site `{site_id}`", + app.control.controller_plan.site_id )) - })?; - if let Ok(protocol_error) = serde_json::from_slice::(&body) { - return Err(protocol_error.into()); - } - Err(ProtocolApiError::control_state_unavailable(format!( - "site controller for site `{site_id}` returned {status} through site router" - ))) + }) } -async fn peer_controller_get_via_router( +async fn remote_controller_post( app: &SiteControllerApp, site_id: &str, path: &str, + body: &TReq, ) -> std::result::Result { let response = app .control .client - .get(format!( + .post(format!( "{}{}", - peer_site_router_url_for_site(app, site_id)?.trim_end_matches('/'), + remote_controller_base_url(app, site_id)?.trim_end_matches('/'), path )) - .header(CONTROLLER_LOCAL_ONLY_HEADER, "1") + .json(body) .send() .await .map_err(|err| { ProtocolApiError::control_state_unavailable(format!( - "failed to reach site controller for site `{site_id}` through site router: {err}" + "failed to reach site controller for site `{site_id}` through its router-local \ + controller route: {err}" )) })?; if response.status().is_success() { return response.json().await.map_err(|err| { ProtocolApiError::control_state_unavailable(format!( - "site controller for site `{site_id}` returned invalid JSON through site router: \ - {err}" + "site controller for site `{site_id}` returned invalid JSON through its \ + router-local controller route: {err}" )) }); } @@ -492,36 +550,11 @@ async fn peer_controller_get_via_router( return Err(protocol_error.into()); } Err(ProtocolApiError::control_state_unavailable(format!( - "site controller for site `{site_id}` returned {status} through site router" + "site controller for site `{site_id}` returned {status} through its router-local \ + controller route" ))) } -pub(super) async fn peer_router_identity_via_router( - app: &SiteControllerApp, - site_id: &str, -) -> std::result::Result { - let response: RouterIdentityResponse = - peer_controller_get_via_router(app, site_id, "/v1/controller/router-identity").await?; - let public_key = base64::engine::general_purpose::STANDARD - .decode(response.public_key_b64) - .map_err(|err| { - ProtocolApiError::control_state_unavailable(format!( - "site controller for site `{site_id}` returned an invalid router public key: {err}" - )) - })?; - let public_key: [u8; 32] = public_key.try_into().map_err(|_| { - ProtocolApiError::control_state_unavailable(format!( - "site controller for site `{site_id}` returned a router public key with the wrong \ - length" - )) - })?; - Ok(MeshIdentityPublic { - id: response.id, - public_key, - mesh_scope: Some(app.control.mesh_scope.to_string()), - }) -} - async fn local_held_list( app: &SiteControllerApp, request: ControlDynamicHeldListRequest, @@ -663,10 +696,10 @@ async fn sync_shared_grant_authority_site( .get(grant_id) .ok_or_else(|| { ProtocolApiError::control_state_unavailable(format!( - "dynamic capability share reported grant `{grant_id}` but it is missing from \ - site controller `{}` state", - app.control.controller_plan.site_id - )) + "dynamic capability share reported grant `{grant_id}` but it is missing \ + from site controller `{}` state", + app.control.controller_plan.site_id + )) })?; site_id_for_logical_component(&state, &grant.holder_component_id).map_err(|err| { ProtocolApiError::control_state_unavailable(format!( @@ -679,7 +712,7 @@ async fn sync_shared_grant_authority_site( if holder_site_id == authority_site_id { return Ok(()); } - let _: ControlDynamicGrantAuthoritySyncResponse = peer_dynamic_caps_post_via_router( + let _: ControlDynamicGrantAuthoritySyncResponse = remote_controller_post( app, &holder_site_id, "/v1/controller/dynamic-caps/grant-authorities/sync", @@ -826,9 +859,9 @@ pub(super) async fn execute_site_controller_dynamic_caps_inspect( )); } let mut held = local_held_list(app, request.clone()).await?.held; - for site_id in peer_site_ids(app) { + for site_id in remote_controller_urls(app).into_keys() { held.extend( - peer_dynamic_caps_post_via_router::<_, HeldListResponse>( + remote_controller_post::<_, HeldListResponse>( app, &site_id, "/v1/controller/dynamic-caps/held", @@ -858,7 +891,7 @@ pub(super) async fn execute_site_controller_dynamic_caps_inspect( )); } Ok(DynamicCapsInspectResponse::HeldDetail( - peer_dynamic_caps_post_via_router( + remote_controller_post( app, &site_id, "/v1/controller/dynamic-caps/held/detail", @@ -881,7 +914,7 @@ pub(super) async fn execute_site_controller_dynamic_caps_inspect( )); } Ok(DynamicCapsInspectResponse::InspectRef( - peer_dynamic_caps_post_via_router( + remote_controller_post( app, &site_id, "/v1/controller/dynamic-caps/inspect-ref", @@ -911,7 +944,7 @@ pub(super) async fn execute_site_controller_dynamic_caps_inspect( .mesh_config .identity; Ok(DynamicCapsInspectResponse::ResolveOrigin( - peer_dynamic_caps_post_via_router( + remote_controller_post( app, &site_id, "/v1/internal/dynamic-caps/resolve-origin", @@ -951,7 +984,7 @@ pub(super) async fn execute_site_controller_dynamic_caps_mutate( )); } Ok(DynamicCapsMutateResponse::Share( - peer_dynamic_caps_post_via_router( + remote_controller_post( app, &site_id, "/v1/controller/dynamic-caps/share", @@ -974,7 +1007,7 @@ pub(super) async fn execute_site_controller_dynamic_caps_mutate( )); } Ok(DynamicCapsMutateResponse::Revoke( - peer_dynamic_caps_post_via_router( + remote_controller_post( app, &site_id, "/v1/controller/dynamic-caps/revoke", @@ -1139,7 +1172,7 @@ async fn publish_dynamic_origin( Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_local_controller_request(&app, &headers)?; + authorize_remote_controller_request(&app, &headers).await?; let ccs_app = LocalDynamicCapabilityOriginApp { site_state_root: PathBuf::from(&app.control.controller_plan.site_state_root), runtime: app.control.runtime.clone(), @@ -1155,7 +1188,7 @@ async fn resolve_dynamic_origin_internal_route( Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_local_controller_request(&app, &headers)?; + authorize_remote_controller_request(&app, &headers).await?; Ok(Json( resolve_dynamic_capability_origin_internal(&app.control, request).await?, )) @@ -1166,7 +1199,7 @@ async fn revoke_dynamic_origin_overlays( headers: HeaderMap, Json(request): Json, ) -> std::result::Result { - authorize_local_controller_request(&app, &headers)?; + authorize_remote_controller_request(&app, &headers).await?; revoke_dynamic_capability_origin_overlays_local(&app.control, &request) .await .map_err(ProtocolApiError::from)?; @@ -1178,7 +1211,7 @@ async fn resolve_external_link_url( headers: HeaderMap, Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_local_controller_request(&app, &headers)?; + authorize_remote_controller_request(&app, &headers).await?; Ok(Json( resolve_external_link_url_local(&app.control, &request) .await @@ -1191,12 +1224,16 @@ async fn prepare_child_runtime_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result { - authorize_local_controller_request(&app, &headers)?; + authorize_remote_controller_request(&app, &headers).await?; ensure_controller_ready(&app)?; - let site_id = app.control.controller_plan.site_id.clone(); - prepare_child_on_local_site(&app.control, &request.state, &request.child, &site_id) - .await - .map_err(ProtocolApiError::from)?; + prepare_child_on_local_site( + &app.control, + &request.state, + &request.child, + &request.site_id, + ) + .await + .map_err(ProtocolApiError::from)?; Ok(StatusCode::NO_CONTENT) } @@ -1205,12 +1242,16 @@ async fn publish_child_runtime_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result { - authorize_local_controller_request(&app, &headers)?; + authorize_remote_controller_request(&app, &headers).await?; ensure_controller_ready(&app)?; - let site_id = app.control.controller_plan.site_id.clone(); - publish_child_on_local_site(&app.control, &request.state, &request.child, &site_id) - .await - .map_err(ProtocolApiError::from)?; + publish_child_on_local_site( + &app.control, + &request.state, + &request.child, + &request.site_id, + ) + .await + .map_err(ProtocolApiError::from)?; Ok(StatusCode::NO_CONTENT) } @@ -1219,10 +1260,9 @@ async fn rollback_child_runtime_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result { - authorize_local_controller_request(&app, &headers)?; + authorize_remote_controller_request(&app, &headers).await?; ensure_controller_ready(&app)?; - let site_id = app.control.controller_plan.site_id.clone(); - rollback_child_on_site(&app.control, request.child_id, &site_id) + rollback_child_on_site(&app.control, request.child_id, &request.site_id) .await .map_err(|err| ProtocolApiError::control_state_unavailable(err.to_string()))?; Ok(StatusCode::NO_CONTENT) @@ -1233,12 +1273,16 @@ async fn destroy_child_runtime_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result { - authorize_local_controller_request(&app, &headers)?; + authorize_remote_controller_request(&app, &headers).await?; ensure_controller_ready(&app)?; - let site_id = app.control.controller_plan.site_id.clone(); - destroy_child_on_local_site(&app.control, &request.state, &request.child, &site_id) - .await - .map_err(ProtocolApiError::from)?; + destroy_child_on_local_site( + &app.control, + &request.state, + &request.child, + &request.site_id, + ) + .await + .map_err(ProtocolApiError::from)?; Ok(StatusCode::NO_CONTENT) } @@ -1247,7 +1291,7 @@ async fn publish_external_slot_overlay_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result { - authorize_local_controller_request(&app, &headers)?; + authorize_remote_controller_request(&app, &headers).await?; publish_external_slot_overlay_local(&app.control, &request) .await .map_err(ProtocolApiError::from)?; @@ -1259,7 +1303,7 @@ async fn clear_external_slot_overlay_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result { - authorize_local_controller_request(&app, &headers)?; + authorize_remote_controller_request(&app, &headers).await?; clear_external_slot_overlay_local(&app.control, &request) .await .map_err(ProtocolApiError::from)?; @@ -1271,7 +1315,7 @@ async fn publish_export_peer_overlay_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result { - authorize_local_controller_request(&app, &headers)?; + authorize_remote_controller_request(&app, &headers).await?; publish_export_peer_overlay_local(&app.control, &request) .await .map_err(ProtocolApiError::from)?; @@ -1283,7 +1327,7 @@ async fn clear_export_peer_overlay_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result { - authorize_local_controller_request(&app, &headers)?; + authorize_remote_controller_request(&app, &headers).await?; clear_export_peer_overlay_local(&app.control, &request) .await .map_err(ProtocolApiError::from)?; @@ -1295,11 +1339,11 @@ async fn control_dynamic_held_list_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_dynamic_caps_sidecar_request(&app, &headers, &request.holder_component_id)?; + let auth = authorize_dynamic_caps_request(&app, &headers, &request.holder_component_id).await?; match execute_site_controller_dynamic_caps_inspect( &app, DynamicCapsInspectRequest::HeldList(request), - controller_local_only(&headers), + matches!(auth, DynamicCapsRequestAuth::RemoteController), ) .await? { @@ -1313,11 +1357,11 @@ async fn control_dynamic_held_detail_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_dynamic_caps_sidecar_request(&app, &headers, &request.holder_component_id)?; + let auth = authorize_dynamic_caps_request(&app, &headers, &request.holder_component_id).await?; match execute_site_controller_dynamic_caps_inspect( &app, DynamicCapsInspectRequest::HeldDetail(request), - controller_local_only(&headers), + matches!(auth, DynamicCapsRequestAuth::RemoteController), ) .await? { @@ -1331,11 +1375,11 @@ async fn control_dynamic_share_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_dynamic_caps_sidecar_request(&app, &headers, &request.caller_component_id)?; + let auth = authorize_dynamic_caps_request(&app, &headers, &request.caller_component_id).await?; match execute_site_controller_dynamic_caps_mutate( &app, DynamicCapsMutateRequest::Share(request), - controller_local_only(&headers), + matches!(auth, DynamicCapsRequestAuth::RemoteController), ) .await? { @@ -1349,7 +1393,7 @@ async fn control_dynamic_grant_authorities_sync_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_local_controller_request(&app, &headers)?; + authorize_remote_controller_request(&app, &headers).await?; Ok(Json( local_sync_dynamic_grant_authorities(&app, request).await?, )) @@ -1360,11 +1404,11 @@ async fn control_dynamic_inspect_ref_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_dynamic_caps_sidecar_request(&app, &headers, &request.holder_component_id)?; + let auth = authorize_dynamic_caps_request(&app, &headers, &request.holder_component_id).await?; match execute_site_controller_dynamic_caps_inspect( &app, DynamicCapsInspectRequest::InspectRef(request), - controller_local_only(&headers), + matches!(auth, DynamicCapsRequestAuth::RemoteController), ) .await? { @@ -1378,11 +1422,11 @@ async fn control_dynamic_revoke_route( headers: HeaderMap, Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_dynamic_caps_sidecar_request(&app, &headers, &request.caller_component_id)?; + let auth = authorize_dynamic_caps_request(&app, &headers, &request.caller_component_id).await?; match execute_site_controller_dynamic_caps_mutate( &app, DynamicCapsMutateRequest::Revoke(request), - controller_local_only(&headers), + matches!(auth, DynamicCapsRequestAuth::RemoteController), ) .await? { @@ -1397,11 +1441,11 @@ async fn control_dynamic_resolve_origin_route( Json(request): Json, ) -> std::result::Result, ProtocolApiError> { - authorize_dynamic_caps_sidecar_request(&app, &headers, &request.holder_component_id)?; + let auth = authorize_dynamic_caps_request(&app, &headers, &request.holder_component_id).await?; match execute_site_controller_dynamic_caps_inspect( &app, DynamicCapsInspectRequest::ResolveOrigin(request), - controller_local_only(&headers), + matches!(auth, DynamicCapsRequestAuth::RemoteController), ) .await? { diff --git a/runtime/site-controller/src/state.rs b/runtime/site-controller/src/state.rs index 0083940e..89aa3797 100644 --- a/runtime/site-controller/src/state.rs +++ b/runtime/site-controller/src/state.rs @@ -9,7 +9,8 @@ pub(super) const SITE_CONTROLLER_STATE_PATH: &str = "/v1/controller/state"; pub(super) const FRAMEWORK_ROUTE_ID_HEADER: &str = "x-amber-route-id"; pub(super) const FRAMEWORK_PEER_ID_HEADER: &str = "x-amber-peer-id"; pub(super) const CONTROL_STATE_AUTH_HEADER: &str = "x-amber-control-state-auth"; -pub const SITE_CONTROLLER_INTERNAL_CAPABILITY: &str = "amber.internal.site_controller"; +pub const SITE_CONTROLLER_INTERNAL_CAPABILITY: &str = + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME; pub const SITE_CONTROLLER_SERVICE_NAME: &str = "amber-site-controller"; pub fn site_controller_internal_route_id(site_id: &str) -> String { @@ -116,19 +117,6 @@ pub struct DynamicProxyExportRecord { pub target_port: u16, } -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct DynamicInputRouteRecord { - pub component: String, - pub slot: String, - pub provider_component: String, - pub protocol: String, - pub capability_kind: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub capability_profile: Option, - #[serde(flatten)] - pub target: DynamicInputRouteTarget, -} - #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "route_kind", rename_all = "snake_case")] pub enum DynamicInputRouteTarget { diff --git a/runtime/site-controller/src/tests.rs b/runtime/site-controller/src/tests.rs index 978cd76d..f2bc7fe3 100644 --- a/runtime/site-controller/src/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -27,7 +27,6 @@ use super::{api::*, http::*, orchestration::*, planner::*, state::*, *}; use crate::{ ccs_api::FrameworkComponentInspectRequest, runtime_api::{SharedSiteControllerRuntime, SiteControllerRuntime}, - site_controller::RouterIdentityResponse, }; #[derive(Clone, Debug, Default, Serialize, Deserialize)] @@ -81,6 +80,7 @@ impl SiteControllerRuntime for TestSiteControllerRuntime { _plan: &'a SiteControllerPlan, _state: FrameworkControlState, _child: LiveChildRecord, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -90,6 +90,7 @@ impl SiteControllerRuntime for TestSiteControllerRuntime { _plan: &'a SiteControllerPlan, _state: FrameworkControlState, _child: LiveChildRecord, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -98,6 +99,7 @@ impl SiteControllerRuntime for TestSiteControllerRuntime { &'a self, _plan: &'a SiteControllerPlan, _child_id: u64, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -107,6 +109,7 @@ impl SiteControllerRuntime for TestSiteControllerRuntime { _plan: &'a SiteControllerPlan, _state: FrameworkControlState, _child: LiveChildRecord, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -275,6 +278,7 @@ impl SiteControllerRuntime for FailingPublishRuntime { _plan: &'a SiteControllerPlan, _state: FrameworkControlState, _child: LiveChildRecord, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -284,6 +288,7 @@ impl SiteControllerRuntime for FailingPublishRuntime { _plan: &'a SiteControllerPlan, _state: FrameworkControlState, _child: LiveChildRecord, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Err(miette::miette!("publish exploded")) }) } @@ -292,6 +297,7 @@ impl SiteControllerRuntime for FailingPublishRuntime { &'a self, _plan: &'a SiteControllerPlan, _child_id: u64, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -301,6 +307,7 @@ impl SiteControllerRuntime for FailingPublishRuntime { _plan: &'a SiteControllerPlan, _state: FrameworkControlState, child: LiveChildRecord, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { let destroy_calls = self.destroy_calls.clone(); Box::pin(async move { @@ -405,6 +412,7 @@ impl SiteControllerRuntime for RecordingPublishRuntime { _plan: &'a SiteControllerPlan, _state: FrameworkControlState, _child: LiveChildRecord, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -414,6 +422,7 @@ impl SiteControllerRuntime for RecordingPublishRuntime { _plan: &'a SiteControllerPlan, _state: FrameworkControlState, child: LiveChildRecord, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { let publish_calls = self.publish_calls.clone(); Box::pin(async move { @@ -430,6 +439,7 @@ impl SiteControllerRuntime for RecordingPublishRuntime { &'a self, _plan: &'a SiteControllerPlan, _child_id: u64, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -439,6 +449,7 @@ impl SiteControllerRuntime for RecordingPublishRuntime { _plan: &'a SiteControllerPlan, _state: FrameworkControlState, _child: LiveChildRecord, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -565,6 +576,7 @@ impl SiteControllerRuntime for PublishBeforeResolveRuntime { _plan: &'a SiteControllerPlan, _state: FrameworkControlState, _child: LiveChildRecord, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -574,6 +586,7 @@ impl SiteControllerRuntime for PublishBeforeResolveRuntime { _plan: &'a SiteControllerPlan, _state: FrameworkControlState, child: LiveChildRecord, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { let publish_calls = self.publish_calls.clone(); Box::pin(async move { @@ -591,6 +604,7 @@ impl SiteControllerRuntime for PublishBeforeResolveRuntime { &'a self, _plan: &'a SiteControllerPlan, _child_id: u64, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -600,6 +614,7 @@ impl SiteControllerRuntime for PublishBeforeResolveRuntime { _plan: &'a SiteControllerPlan, _state: FrameworkControlState, _child: LiveChildRecord, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -704,6 +719,7 @@ impl SiteControllerRuntime for FailingRollbackRuntime { _plan: &'a SiteControllerPlan, _state: FrameworkControlState, _child: LiveChildRecord, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -713,6 +729,7 @@ impl SiteControllerRuntime for FailingRollbackRuntime { _plan: &'a SiteControllerPlan, _state: FrameworkControlState, _child: LiveChildRecord, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -721,6 +738,7 @@ impl SiteControllerRuntime for FailingRollbackRuntime { &'a self, _plan: &'a SiteControllerPlan, _child_id: u64, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Err(miette::miette!("rollback exploded")) }) } @@ -730,6 +748,7 @@ impl SiteControllerRuntime for FailingRollbackRuntime { _plan: &'a SiteControllerPlan, _state: FrameworkControlState, _child: LiveChildRecord, + _site_id: &'a str, ) -> SiteControllerRuntimeFuture<'a, ()> { Box::pin(async { Ok(()) }) } @@ -1061,7 +1080,7 @@ fn delegated_entry_for( .expect("delegated held entry should resolve") } -async fn compile_dynamic_caps_binding_state() -> FrameworkControlState { +fn dynamic_caps_binding_scenario_ir() -> ScenarioIr { fn path_program() -> amber_scenario::Program { serde_json::from_value(serde_json::json!({ "path": "/usr/bin/env", @@ -1121,7 +1140,7 @@ async fn compile_dynamic_caps_binding_state() -> FrameworkControlState { } } - let scenario = ScenarioIr { + ScenarioIr { schema: amber_scenario::SCENARIO_IR_SCHEMA.to_string(), version: amber_scenario::SCENARIO_IR_VERSION, root: 0, @@ -1218,7 +1237,11 @@ async fn compile_dynamic_caps_binding_state() -> FrameworkControlState { ], exports: Vec::new(), manifest_catalog: BTreeMap::new(), - }; + } +} + +async fn compile_dynamic_caps_binding_state() -> FrameworkControlState { + let scenario = dynamic_caps_binding_scenario_ir(); let placement = PlacementFile { schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), version: amber_compiler::run_plan::PLACEMENT_VERSION, @@ -1830,7 +1853,6 @@ async fn same_site_dynamic_child_output_bindings_reuse_provider_component_routes let runtime_spec = build_local_child_runtime_spec(&state, consumer, &site_id).expect("runtime spec"); assert_eq!(runtime_spec.direct_inputs.len(), 1); - assert!(runtime_spec.routed_inputs.is_empty()); assert_eq!(runtime_spec.direct_inputs[0].component, "/consumer"); assert_eq!(runtime_spec.direct_inputs[0].slot, "required_api"); assert_eq!( @@ -1976,7 +1998,6 @@ async fn same_site_static_child_export_bindings_reuse_provider_component_routes( let runtime_spec = build_local_child_runtime_spec(&state, consumer, &site_id).expect("runtime spec"); assert_eq!(runtime_spec.direct_inputs.len(), 1); - assert!(runtime_spec.routed_inputs.is_empty()); assert_eq!(runtime_spec.direct_inputs[0].component, "/consumer"); assert_eq!(runtime_spec.direct_inputs[0].slot, "upstream"); assert_eq!( @@ -2050,9 +2071,9 @@ async fn dynamic_grant_routes_to_holder_site_not_offered_site_order() { } #[tokio::test] -async fn dynamic_caps_cross_site_share_syncs_holder_authority_through_site_router() { +async fn dynamic_caps_cross_site_share_syncs_holder_authority_through_remote_controller_capability() +{ let dir = TempDir::new().expect("temp dir"); - let base = compile_dynamic_caps_binding_state().await; let placement = PlacementFile { schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), version: amber_compiler::run_plan::PLACEMENT_VERSION, @@ -2087,7 +2108,7 @@ async fn dynamic_caps_cross_site_share_syncs_holder_authority_through_site_route framework_children: None, }; let mut authority_state = compile_control_state_from_ir_with_run_id( - base.base_scenario.clone(), + dynamic_caps_binding_scenario_ir(), Some(&placement), "test-run", ) @@ -2115,17 +2136,10 @@ async fn dynamic_caps_cross_site_share_syncs_holder_authority_through_site_route "/v1/controller/dynamic-caps/grant-authorities/sync", axum::routing::post({ let sync_requests = sync_requests.clone(); - move |headers: HeaderMap, + move |_headers: HeaderMap, Json(request): Json| { let sync_requests = sync_requests.clone(); async move { - assert_eq!( - headers - .get(super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER) - .and_then(|value| value.to_str().ok()), - Some("1"), - "cross-site share sync should stay local on the destination controller", - ); sync_requests .lock() .expect("sync request log poisoned") @@ -2143,8 +2157,10 @@ async fn dynamic_caps_cross_site_share_syncs_holder_authority_through_site_route let controller_plan = Arc::make_mut(&mut app.controller_plan); controller_plan.site_id = "direct_a".to_string(); controller_plan.router_identity_id = "/site/direct_a/router".to_string(); - controller_plan.peer_site_router_urls = - BTreeMap::from([("direct_b".to_string(), holder_base_url)]); + install_remote_controller_peer_url_fixture( + &mut app, + &BTreeMap::from([("direct_b".to_string(), holder_base_url)]), + ); let controller_app = SiteControllerApp { control: app, ready: ready_site_controller_flag(), @@ -2452,6 +2468,77 @@ async fn compile_framework_binding_control_state() -> ( (dir, state, state_path, record) } +async fn compile_cross_site_framework_authority_control_state() +-> (TempDir, FrameworkControlState, PathBuf) { + let dir = TempDir::new().expect("temp dir"); + let admin_path = dir.path().join("admin.json5"); + let authority_path = dir.path().join("authority.json5"); + let root_path = dir.path().join("root.json5"); + write_file( + &admin_path, + r#" + { + manifest_version: "0.3.0", + slots: { ctl: { kind: "component" } }, + program: { + path: "/usr/bin/env", + args: ["sh", "-c", "sleep 30"], + env: { CTL_URL: "${slots.ctl.url}" }, + network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] } + }, + provides: { api: { kind: "http", endpoint: "http" } }, + exports: { api: "api" }, + } + "#, + ); + write_file( + &authority_path, + &format!( + r##" + {{ + manifest_version: "0.3.0", + components: {{ admin: "{admin}" }}, + bindings: [ + {{ to: "#admin.ctl", from: "framework.component" }} + ], + program: {{ + image: "busybox:1.36.1", + entrypoint: ["sh", "-c", "sleep 30"], + network: {{ endpoints: [{{ name: "http", port: 8081, protocol: "http" }}] }} + }}, + provides: {{ api: {{ kind: "http", endpoint: "http" }} }}, + exports: {{ + api: "api", + admin_api: "#admin.api" + }}, + }} + "##, + admin = file_url(&admin_path), + ), + ); + write_file( + &root_path, + &format!( + r##" + {{ + manifest_version: "0.3.0", + components: {{ authority: "{authority}" }}, + exports: {{ + authority_api: "#authority.api", + authority_admin_api: "#authority.admin_api" + }}, + }} + "##, + authority = file_url(&authority_path), + ), + ); + + let state = compile_control_state(&root_path).await; + let state_path = dir.path().join("control-state.json"); + write_control_state(&state_path, &state).expect("state should write"); + (dir, state, state_path) +} + fn empty_live_child( authority_realm_id: usize, name: &str, @@ -2587,10 +2674,13 @@ fn test_control_state_app( }; let router_control = test_router_control_addr(); let router_public_key_b64 = base64::engine::general_purpose::STANDARD.encode([7u8; 32]); - let site_id = offered_sites - .keys() - .next() + let site_id = state + .placement + .assignments + .get("/") .cloned() + .or_else(|| state.placement.defaults.path.clone()) + .or_else(|| offered_sites.keys().next().cloned()) .expect("offered sites should contain at least one site"); for (site_id, site_definition) in &offered_sites { let site_state_root = state_root.join(site_id); @@ -2938,6 +3028,14 @@ async fn install_framework_site_controller_fixture( vec![overlay_handle] } +fn install_remote_controller_peer_url_fixture( + app: &mut ControlStateApp, + remote_base_urls: &BTreeMap, +) { + let plan = Arc::make_mut(&mut app.controller_plan); + plan.peer_site_router_urls.extend(remote_base_urls.clone()); +} + struct TestMcpClient { client: Client, endpoint: String, @@ -3518,7 +3616,7 @@ async fn refresh_capability_instances_rejects_unlowered_framework_component_bind ); } #[tokio::test] -async fn recover_control_state_reconciles_framework_routes_without_live_peer_controllers() { +async fn recover_control_state_reconciles_framework_routes_without_live_remote_controllers() { let (dir, mut state, state_path, _) = compile_framework_binding_control_state().await; state.placement.offered_sites = BTreeMap::from([ ( @@ -3568,7 +3666,7 @@ async fn recover_control_state_reconciles_framework_routes_without_live_peer_con }; recover_control_state(&authority_app).await.expect( - "recovery should not require a live peer controller when peer router identities were \ + "recovery should not require a live remote controller when peer router identities were \ already planned", ); } @@ -3710,6 +3808,14 @@ fn inject_site_controller_peer_router_routes_records_peer_router_identities() { "the injected router plan should terminate peer site-controller traffic at the lowered \ controller component", ); + assert!( + router + .config + .peers + .iter() + .any(|peer| peer.id == "/site/local/controller"), + "the router plan should trust the lowered local controller peer it forwards to", + ); assert!( router .config @@ -3721,66 +3827,6 @@ fn inject_site_controller_peer_router_routes_records_peer_router_identities() { ); } -#[tokio::test] -async fn prepare_child_on_site_rejects_nonlocal_site_plan() { - let dir = TempDir::new().expect("temp dir"); - let root_path = dir.path().join("root.json5"); - let worker_path = dir.path().join("worker.json5"); - write_file( - &worker_path, - r#" - { - manifest_version: "0.3.0", - program: { - path: "/bin/echo", - args: ["worker"], - network: { endpoints: [{ name: "http", port: 8080 }] } - } - } - "#, - ); - write_file( - &root_path, - &format!( - r#" - {{ - manifest_version: "0.3.0", - program: {{ path: "/bin/echo", args: ["root"] }}, - slots: {{ - realm: {{ kind: "component", optional: true }} - }}, - child_templates: {{ - worker: {{ - manifest: "{worker}" - }} - }} - }} - "#, - worker = file_url(&worker_path), - ), - ); - let state = compile_control_state(&root_path).await; - let state_path = dir.path().join("control-state.json"); - write_control_state(&state_path, &state).expect("state should write"); - let app = test_control_state_app(&dir, state, state_path); - let mut child = empty_live_child(1, "remote", 1, ChildState::CreatePrepared); - child.assignments = BTreeMap::from([("/remote".to_string(), "other-site".to_string())]); - let err = prepare_child_on_site( - &app, - &app.control_state.lock().await.clone(), - &child, - "other-site", - ) - .await - .expect_err("nonlocal children without a router path should fail"); - assert_eq!(err.code, ProtocolErrorCode::OriginUnavailable); - assert!( - err.message.contains("has no router forward path"), - "unexpected error: {}", - err.message - ); -} - struct DynamicCapsMcpHarness { _dir: TempDir, client: Client, @@ -4089,7 +4135,7 @@ async fn dynamic_caps_held_list_ignores_unrouted_offered_sites() { false, ) .await - .expect("held list should stay local when no peer controller routes exist"); + .expect("held list should stay local when no remote controller routes exist"); let super::control_state_api::DynamicCapsInspectResponse::HeldList(held) = response else { panic!("held list request should return a held list response"); @@ -4104,7 +4150,6 @@ async fn dynamic_caps_held_list_ignores_unrouted_offered_sites() { #[tokio::test] async fn localize_framework_control_state_tracks_remote_grant_authority_sites() { - let base = compile_dynamic_caps_binding_state().await; let placement = PlacementFile { schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), version: amber_compiler::run_plan::PLACEMENT_VERSION, @@ -4139,7 +4184,7 @@ async fn localize_framework_control_state_tracks_remote_grant_authority_sites() framework_children: None, }; let mut state = compile_control_state_from_ir_with_run_id( - base.base_scenario.clone(), + dynamic_caps_binding_scenario_ir(), Some(&placement), "test-run", ) @@ -4182,7 +4227,6 @@ async fn localize_framework_control_state_tracks_remote_grant_authority_sites() #[tokio::test] async fn inspect_ref_routes_remote_grants_via_synced_authority_site() { let dir = TempDir::new().expect("temp dir"); - let base = compile_dynamic_caps_binding_state().await; let placement = PlacementFile { schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), version: amber_compiler::run_plan::PLACEMENT_VERSION, @@ -4217,7 +4261,7 @@ async fn inspect_ref_routes_remote_grants_via_synced_authority_site() { framework_children: None, }; let mut authoritative = compile_control_state_from_ir_with_run_id( - base.base_scenario.clone(), + dynamic_caps_binding_scenario_ir(), Some(&placement), "test-run", ) @@ -4257,18 +4301,11 @@ async fn inspect_ref_routes_remote_grants_via_synced_authority_site() { axum::routing::post({ let hits = hits.clone(); let remote_grant_id = remote_grant_id.clone(); - move |headers: HeaderMap, + move |_headers: HeaderMap, Json(request): Json| { let hits = hits.clone(); let remote_grant_id = remote_grant_id.clone(); async move { - assert_eq!( - headers - .get(super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER) - .and_then(|value| value.to_str().ok()), - Some("1"), - "peer-routed inspect_ref should stay local on the destination controller", - ); hits.lock() .expect("inspect-ref hit log poisoned") .push(request.r#ref.clone()); @@ -4293,8 +4330,10 @@ async fn inspect_ref_routes_remote_grants_via_synced_authority_site() { let controller_plan = Arc::make_mut(&mut app.controller_plan); controller_plan.site_id = "direct_b".to_string(); controller_plan.router_identity_id = "/site/direct_b/router".to_string(); - controller_plan.peer_site_router_urls = - BTreeMap::from([("direct_a".to_string(), authority_base_url)]); + install_remote_controller_peer_url_fixture( + &mut app, + &BTreeMap::from([("direct_a".to_string(), authority_base_url)]), + ); let controller_app = SiteControllerApp { control: app, ready: ready_site_controller_flag(), @@ -4329,9 +4368,8 @@ async fn inspect_ref_routes_remote_grants_via_synced_authority_site() { } #[tokio::test] -async fn held_list_aggregates_remote_grants_via_peer_router() { +async fn held_list_aggregates_remote_grants_via_remote_controller_capability() { let dir = TempDir::new().expect("temp dir"); - let base = compile_dynamic_caps_binding_state().await; let placement = PlacementFile { schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), version: amber_compiler::run_plan::PLACEMENT_VERSION, @@ -4366,7 +4404,7 @@ async fn held_list_aggregates_remote_grants_via_peer_router() { framework_children: None, }; let mut authoritative = compile_control_state_from_ir_with_run_id( - base.base_scenario.clone(), + dynamic_caps_binding_scenario_ir(), Some(&placement), "test-run", ) @@ -4404,18 +4442,11 @@ async fn held_list_aggregates_remote_grants_via_peer_router() { axum::routing::post({ let hits = hits.clone(); let remote_grant_id = remote_grant_id.clone(); - move |headers: HeaderMap, + move |_headers: HeaderMap, Json(request): Json| { let hits = hits.clone(); let remote_grant_id = remote_grant_id.clone(); async move { - assert_eq!( - headers - .get(super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER) - .and_then(|value| value.to_str().ok()), - Some("1"), - "peer-routed held_list should stay local on the destination controller", - ); hits.lock() .expect("held-list hit log poisoned") .push(request.holder_component_id.clone()); @@ -4445,8 +4476,10 @@ async fn held_list_aggregates_remote_grants_via_peer_router() { let controller_plan = Arc::make_mut(&mut app.controller_plan); controller_plan.site_id = "direct_b".to_string(); controller_plan.router_identity_id = "/site/direct_b/router".to_string(); - controller_plan.peer_site_router_urls = - BTreeMap::from([("direct_a".to_string(), authority_base_url)]); + install_remote_controller_peer_url_fixture( + &mut app, + &BTreeMap::from([("direct_a".to_string(), authority_base_url)]), + ); let controller_app = SiteControllerApp { control: app, ready: ready_site_controller_flag(), @@ -5871,14 +5904,15 @@ async fn delegated_cross_site_framework_requests_route_to_the_forwarded_authorit let router_public_key_b64 = base64::engine::general_purpose::STANDARD.encode([11u8; 32]); let peer_router_attempts = std::sync::Arc::new(std::sync::Mutex::new(0usize)); let peer_router = Router::new().route( - "/v1/controller/router-identity", - axum::routing::get({ + "/v1/internal/router-identity", + axum::routing::post({ let router_public_key_b64 = router_public_key_b64.clone(); let peer_router_attempts = peer_router_attempts.clone(); - move || { + move |Json(request): Json| { let router_public_key_b64 = router_public_key_b64.clone(); let peer_router_attempts = peer_router_attempts.clone(); async move { + assert_eq!(request.site_id, "direct_local"); let mut attempts = peer_router_attempts .lock() .expect("peer router attempt log poisoned"); @@ -6087,15 +6121,33 @@ async fn capability_instance_auth_and_snapshot_scope_are_enforced() { async fn create_child_publishes_sites_before_resolving_cross_site_link_overlays() { let dir = TempDir::new().expect("temp dir"); let root_path = dir.path().join("root.json5"); + let admin_path = dir.path().join("admin.json5"); write_file( - &root_path, + &admin_path, r#" { manifest_version: "0.3.0", - program: { path: "/bin/echo", args: ["root"] } + slots: { ctl: { kind: "component" } }, + program: { path: "/bin/echo", args: ["admin", "${slots.ctl.url}"] } } "#, ); + write_file( + &root_path, + &format!( + r##" + {{ + manifest_version: "0.3.0", + program: {{ path: "/bin/echo", args: ["root"] }}, + components: {{ admin: "{admin}" }}, + bindings: [ + {{ to: "#admin.ctl", from: "framework.component" }} + ], + }} + "##, + admin = file_url(&admin_path), + ), + ); let placement = PlacementFile { schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), @@ -6127,6 +6179,48 @@ async fn create_child_publishes_sites_before_resolving_cross_site_link_overlays( framework_children: None, }; let mut state = compile_control_state_with_placement(&root_path, Some(&placement)).await; + for execution_site in ["direct_local", "compose_local"] { + if state.base_scenario.components.iter().any(|component| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .is_some_and(|metadata| metadata.execution_site == execution_site) + }) { + continue; + } + let mut scenario = Scenario::try_from(state.base_scenario.clone()).expect("base scenario"); + let root_id = scenario.root; + let controller_id = amber_scenario::ComponentId(scenario.components.len()); + scenario.components.push(Some(amber_scenario::Component { + id: controller_id, + parent: Some(root_id), + moniker: format!("/__amber_internal_framework_component_controller/{execution_site}") + .into(), + digest: scenario.component(root_id).digest, + config: None, + config_schema: None, + program: None, + slots: BTreeMap::new(), + provides: BTreeMap::new(), + resources: BTreeMap::new(), + metadata: Some( + serde_json::to_value( + amber_compiler::run_plan::FrameworkComponentControllerMetadata { + kind: "amber.framework_component.controller".to_string(), + execution_site: execution_site.to_string(), + grants: BTreeMap::new(), + }, + ) + .expect("controller metadata should serialize"), + ), + child_templates: BTreeMap::new(), + children: Vec::new(), + })); + scenario.component_mut(root_id).children.push(controller_id); + scenario.normalize_order(); + scenario.assert_invariants(); + state.base_scenario = amber_scenario::ScenarioIr::from(&scenario); + } let state_path = dir.path().join("control-state.json"); let root_authority = state.base_scenario.root; let child_root_id = 10_001usize; @@ -6237,8 +6331,9 @@ async fn create_child_publishes_sites_before_resolving_cross_site_link_overlays( let peer_router = Router::new() .route( - "/v1/controller/router-identity", - axum::routing::get(|| async { + "/v1/internal/router-identity", + axum::routing::post(|Json(request): Json| async move { + assert_eq!(request.site_id, "compose_local"); Json(RouterIdentityResponse { id: "/site/compose_local/router".to_string(), public_key_b64: base64::engine::general_purpose::STANDARD.encode([13u8; 32]), @@ -6282,7 +6377,6 @@ async fn create_child_publishes_sites_before_resolving_cross_site_link_overlays( plan.site_id = "direct_local".to_string(); plan.kind = SiteKind::Direct; plan.router_identity_id = "/site/direct_local/router".to_string(); - plan.peer_site_router_urls = BTreeMap::from([("compose_local".to_string(), peer_router_url)]); plan.site_state_root = dir .path() .join("state") @@ -6309,11 +6403,24 @@ async fn create_child_publishes_sites_before_resolving_cross_site_link_overlays( .display() .to_string(), ); - let app = ControlStateApp { + let mut app = ControlStateApp { controller_plan: Arc::new(plan), runtime, ..base_app }; + install_remote_controller_peer_url_fixture( + &mut app, + &BTreeMap::from([("compose_local".to_string(), peer_router_url.clone())]), + ); + assert_eq!( + app.controller_plan + .peer_site_router_urls + .get("compose_local") + .map(String::as_str), + Some(peer_router_url.as_str()), + "controller runtime should use the router-local peer URL for compose-local controller \ + traffic", + ); continue_create_committed_hidden(&app, child.child_id) .await .expect("create should publish the child before resolving cross-site overlays"); @@ -6559,6 +6666,25 @@ async fn local_controller_requests_require_internal_route_headers_not_framework_ control: test_control_state_app(&dir, state, state_path), ready: ready_site_controller_flag(), }; + let controller_route_id = { + let state = app.control.control_state.lock().await.clone(); + let scenario = Scenario::try_from(state.base_scenario.clone()).expect("base scenario"); + let controller = scenario + .components_iter() + .find(|(_, component)| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .is_some() + }) + .map(|(_, component)| component) + .expect("site controller component should be injected"); + amber_mesh::component_route_id( + controller.moniker.as_str(), + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + amber_mesh::MeshProtocol::Http, + ) + }; let mut headers = HeaderMap::new(); headers.insert( super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER, @@ -6566,7 +6692,7 @@ async fn local_controller_requests_require_internal_route_headers_not_framework_ ); headers.insert( FRAMEWORK_ROUTE_ID_HEADER, - site_controller_internal_route_id(&app.control.controller_plan.site_id) + controller_route_id .parse() .expect("route header should parse"), ); @@ -6579,9 +6705,11 @@ async fn local_controller_requests_require_internal_route_headers_not_framework_ .expect("peer header should parse"), ); - super::site_controller::authorize_local_controller_request(&app, &headers).expect( - "router-local controller requests should authorize without a control-state auth header", - ); + super::site_controller::authorize_local_controller_request(&app, &headers) + .await + .expect( + "router-local controller requests should authorize without a control-state auth header", + ); headers.insert( FRAMEWORK_ROUTE_ID_HEADER, @@ -6590,10 +6718,114 @@ async fn local_controller_requests_require_internal_route_headers_not_framework_ .expect("header should parse"), ); let err = super::site_controller::authorize_local_controller_request(&app, &headers) + .await .expect_err("wrong internal route should be rejected"); assert_eq!(err.0.code, ProtocolErrorCode::Unauthorized); } +#[tokio::test] +async fn local_controller_requests_select_the_controller_for_the_current_site() { + let (dir, mut state, state_path, _) = compile_framework_binding_control_state().await; + let mut scenario = Scenario::try_from(state.base_scenario.clone()).expect("base scenario"); + let root_id = scenario.root; + let compose_controller_id = amber_scenario::ComponentId(scenario.components.len()); + scenario.components.push(Some(amber_scenario::Component { + id: compose_controller_id, + parent: Some(root_id), + moniker: "/__amber_internal_framework_component_controller/compose_local" + .to_string() + .into(), + digest: scenario.component(root_id).digest, + config: None, + config_schema: None, + program: None, + slots: BTreeMap::new(), + provides: BTreeMap::new(), + resources: BTreeMap::new(), + metadata: Some( + serde_json::to_value( + amber_compiler::run_plan::FrameworkComponentControllerMetadata { + kind: "amber.framework_component.controller".to_string(), + execution_site: "compose_local".to_string(), + grants: BTreeMap::new(), + }, + ) + .expect("controller metadata should serialize"), + ), + child_templates: BTreeMap::new(), + children: Vec::new(), + })); + scenario + .component_mut(root_id) + .children + .push(compose_controller_id); + scenario.normalize_order(); + scenario.assert_invariants(); + state.base_scenario = amber_scenario::ScenarioIr::from(&scenario); + + let route_id_for_site = |scenario: &Scenario, execution_site: &str| { + let controller = scenario + .components_iter() + .find(|(_, component)| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .is_some_and(|metadata| metadata.execution_site == execution_site) + }) + .map(|(_, component)| component) + .expect("site controller component should be injected for the site"); + amber_mesh::component_route_id( + controller.moniker.as_str(), + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + amber_mesh::MeshProtocol::Http, + ) + }; + + let direct_route_id = route_id_for_site(&scenario, "direct_local"); + let compose_route_id = route_id_for_site(&scenario, "compose_local"); + + let control = test_control_state_app(&dir, state, state_path.clone()); + let mut controller_plan = control.controller_plan.as_ref().clone(); + controller_plan.site_id = "direct_local".to_string(); + controller_plan.kind = SiteKind::Direct; + controller_plan.router_identity_id = "/site/direct_local/router".to_string(); + let app = SiteControllerApp { + control: ControlStateApp { + controller_plan: Arc::new(controller_plan), + ..control + }, + ready: ready_site_controller_flag(), + }; + + let mut headers = HeaderMap::new(); + headers.insert( + super::site_controller::CONTROLLER_LOCAL_ONLY_HEADER, + "1".parse().expect("header should parse"), + ); + headers.insert( + FRAMEWORK_ROUTE_ID_HEADER, + direct_route_id.parse().expect("route header should parse"), + ); + headers.insert( + FRAMEWORK_PEER_ID_HEADER, + "/site/direct_local/router" + .parse() + .expect("peer header should parse"), + ); + super::site_controller::authorize_local_controller_request(&app, &headers) + .await + .expect("the direct-local controller should accept its own lowered internal route"); + + headers.insert( + FRAMEWORK_ROUTE_ID_HEADER, + compose_route_id.parse().expect("route header should parse"), + ); + let err = super::site_controller::authorize_local_controller_request(&app, &headers) + .await + .expect_err("a different site's controller route should be rejected"); + assert_eq!(err.0.code, ProtocolErrorCode::Unauthorized); +} + #[tokio::test] async fn dynamic_caps_sidecar_requests_reject_ambient_control_state_auth_without_internal_route() { let dir = TempDir::new().expect("temp dir"); @@ -6614,7 +6846,6 @@ async fn dynamic_caps_sidecar_requests_reject_ambient_control_state_auth_without ); let err = super::site_controller::authorize_dynamic_caps_sidecar_request( - &app, &headers, "components./alice", ) @@ -6622,6 +6853,218 @@ async fn dynamic_caps_sidecar_requests_reject_ambient_control_state_auth_without assert_eq!(err.0.code, ProtocolErrorCode::Unauthorized); } +#[tokio::test] +async fn cross_site_framework_capability_records_bind_to_provider_router_peers() { + let (_dir, state, _state_path) = compile_cross_site_framework_authority_control_state().await; + let expected_provider_router = "/site/compose_local/router"; + + let admin_record = state + .capability_instances + .values() + .find(|record| record.recipient_component_moniker == "/authority/admin") + .cloned() + .expect("cross-site admin should receive a framework capability record"); + assert_eq!(admin_record.controller_site_id, "compose_local"); + assert_eq!(admin_record.recipient_site_id, "direct_local"); + assert_eq!(admin_record.recipient_peer_id, expected_provider_router); + authorize_capability_instance( + &state, + &admin_record.cap_instance_id, + expected_provider_router, + ) + .expect("cross-site framework grants should authorize through the provider site router"); + let admin_err = + authorize_capability_instance(&state, &admin_record.cap_instance_id, "/authority/admin") + .expect_err( + "cross-site framework grants must not authorize the remote component directly", + ); + assert_eq!(admin_err.code, ProtocolErrorCode::Unauthorized); + + let remote_record = state + .capability_instances + .values() + .find(|record| { + record.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + && record.controller_site_id == "compose_local" + && record.recipient_site_id == "direct_local" + }) + .cloned() + .expect("compose-local controller should expose the remote controller internal capability"); + assert_eq!(remote_record.recipient_peer_id, expected_provider_router); + authorize_capability_instance( + &state, + &remote_record.cap_instance_id, + expected_provider_router, + ) + .expect( + "cross-site remote controller capabilities should authorize through the provider site \ + router", + ); + let peer_err = authorize_capability_instance( + &state, + &remote_record.cap_instance_id, + remote_record.recipient_component_moniker.as_str(), + ) + .expect_err( + "cross-site remote controller capabilities must not authorize the remote controller \ + component directly", + ); + assert_eq!(peer_err.code, ProtocolErrorCode::Unauthorized); +} + +#[tokio::test] +async fn dynamic_caps_remote_controller_requests_authorize_over_public_remote_controller_capability() + { + let (dir, state, state_path) = compile_cross_site_framework_authority_control_state().await; + let app = SiteControllerApp { + control: test_control_state_app(&dir, state.clone(), state_path), + ready: ready_site_controller_flag(), + }; + let remote_record = state + .capability_instances + .values() + .find(|record| { + record.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + && record.controller_site_id == "compose_local" + && record.recipient_site_id == "direct_local" + }) + .cloned() + .expect("compose-local controller should expose the remote controller internal capability"); + let mut headers = HeaderMap::new(); + headers.insert( + FRAMEWORK_ROUTE_ID_HEADER, + remote_record + .route_id + .parse() + .expect("route header should parse"), + ); + headers.insert( + FRAMEWORK_PEER_ID_HEADER, + remote_record + .recipient_peer_id + .parse() + .expect("peer header should parse"), + ); + + let auth = super::site_controller::authorize_dynamic_caps_request( + &app, + &headers, + "components./remote-holder", + ) + .await + .expect( + "remote dynamic capability requests should authorize through the router-local controller \ + route", + ); + assert_eq!( + auth, + super::site_controller::DynamicCapsRequestAuth::RemoteController + ); +} + +#[tokio::test] +async fn router_identity_lookup_requests_target_site_from_remote_controller_site() { + let dir = TempDir::new().expect("temp dir"); + let root_path = dir.path().join("root.json5"); + write_file( + &root_path, + r#" + { + manifest_version: "0.3.0", + program: { path: "/bin/echo", args: ["root"] }, + } + "#, + ); + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "compose_local".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + controller_site: None, + }, + ), + ( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "vm_local".to_string(), + SiteDefinition { + kind: SiteKind::Vm, + context: None, + controller_site: Some("direct_local".to_string()), + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("compose_local".to_string()), + vm: Some("vm_local".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::new(), + dynamic_capabilities: None, + framework_children: None, + }; + let state = compile_control_state_with_placement(&root_path, Some(&placement)).await; + let state_path = dir.path().join("state.json"); + write_control_state(&state_path, &state).expect("state should write"); + + let requested_sites = Arc::new(std::sync::Mutex::new(Vec::::new())); + let vm_public_key = [23u8; 32]; + let peer_router = Router::new().route( + "/v1/internal/router-identity", + axum::routing::post({ + let requested_sites = requested_sites.clone(); + move |Json(request): Json| { + let requested_sites = requested_sites.clone(); + async move { + requested_sites + .lock() + .expect("requested site log should lock") + .push(request.site_id.clone()); + Json(RouterIdentityResponse { + id: format!("/site/{}/router", request.site_id), + public_key_b64: base64::engine::general_purpose::STANDARD + .encode(vm_public_key), + }) + } + } + }), + ); + let (direct_controller_url, _direct_controller_handle) = spawn_test_router(peer_router).await; + + let mut app = test_control_state_app(&dir, state, state_path); + let plan = Arc::make_mut(&mut app.controller_plan); + plan.site_id = "compose_local".to_string(); + plan.kind = SiteKind::Compose; + plan.peer_site_router_urls = BTreeMap::from([( + "direct_local".to_string(), + direct_controller_url.to_string(), + )]); + + let identity = router_identity_for_overlay(&app, "vm_local") + .await + .expect("remote controller site should return the target VM router identity"); + assert_eq!(identity.id, "/site/vm_local/router"); + assert_eq!(identity.public_key, vm_public_key); + assert_eq!( + requested_sites + .lock() + .expect("requested site log should lock") + .clone(), + vec!["vm_local".to_string()], + "the remote controller request must name the target site, not the controller site", + ); +} + #[tokio::test] async fn dynamic_authority_templates_are_listed_and_created_from_live_realm() { let dir = TempDir::new().expect("temp dir"); @@ -7550,6 +7993,43 @@ async fn prepare_child_record_uses_frozen_dynamic_placement_assignments() { ); } +#[tokio::test] +async fn prepare_child_record_grants_internal_controller_route_only_to_dynamic_child() { + let (_dir, mut state, _state_path, _admin_record) = + compile_framework_binding_control_state().await; + let root_authority = state.base_scenario.root; + let child = prepare_child_record( + &mut state, + root_authority, + &CreateChildRequest { + template: "worker".to_string(), + name: "job".to_string(), + manifest: None, + config: BTreeMap::new(), + bindings: BTreeMap::new(), + }, + ) + .await + .expect("dynamic child should plan successfully"); + let spec = build_local_child_runtime_spec(&state, &child, "direct_local") + .expect("dynamic child should build a direct runtime spec"); + let internal_routes = spec + .controller_routes + .iter() + .filter(|route| { + route.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME + }) + .collect::>(); + + assert_eq!(internal_routes.len(), 1); + assert_eq!( + internal_routes[0].allowed_issuers, + vec!["/job".to_string()], + "dynamic child sidecars need the local controller route for dynamic-capability control, \ + but the overlay must not grant access to pre-existing components or unrelated children", + ); +} + #[tokio::test] async fn prepare_child_record_supports_cross_site_dynamic_fragments() { let dir = TempDir::new().expect("temp dir"); @@ -7788,23 +8268,68 @@ async fn prepare_child_record_supports_cross_site_dynamic_fragments() { let compose_spec = build_local_child_runtime_spec(&state, &child, "compose_local").expect("compose spec"); assert_eq!(compose_spec.assigned_components, vec!["/job-compose/root"]); + assert!( + compose_spec.direct_inputs.is_empty(), + "cross-site child bindings should not be modeled as same-site direct inputs" + ); let direct_spec = build_local_child_runtime_spec(&state, &child, "direct_local").expect("direct spec"); assert_eq!( direct_spec.assigned_components, vec!["/job-compose/direct_helper"] ); + assert!( + direct_spec.direct_inputs.is_empty(), + "provider-only sites should not receive synthetic direct inputs for cross-site links" + ); let kind_spec = build_local_child_runtime_spec(&state, &child, "kind_local").expect("kind spec"); assert_eq!( kind_spec.assigned_components, vec!["/job-compose/kind_helper"] ); + assert!( + kind_spec.direct_inputs.is_empty(), + "provider-only sites should not receive synthetic direct inputs for cross-site links" + ); let vm_spec = build_local_child_runtime_spec(&state, &child, "vm_local").expect("vm spec"); assert_eq!( vm_spec.assigned_components, vec!["/job-compose/vm_helper/root"] ); + assert!( + vm_spec.direct_inputs.is_empty(), + "provider-only sites should not receive synthetic direct inputs for cross-site links" + ); + assert_eq!( + child.overlays.len(), + 6, + "three cross-site child bindings should lower to three consumer external-slot overlays and three provider export-peer overlays" + ); + assert_eq!( + child + .overlays + .iter() + .filter(|overlay| { + overlay.site_id == "compose_local" + && matches!(overlay.action, DynamicOverlayAction::ExternalSlot { .. }) + }) + .count(), + 3, + "the child root site should receive one external-slot overlay per cross-site binding" + ); + assert_eq!( + child + .overlays + .iter() + .filter(|overlay| { + overlay.site_id != "compose_local" + && matches!(overlay.action, DynamicOverlayAction::ExportPeer { .. }) + }) + .count(), + 3, + "each remote provider site should expose one export-peer overlay for the child binding" + ); } #[tokio::test] @@ -9351,12 +9876,16 @@ volumes: .iter() .any(|value| { value.as_str() == Some("/var/run/docker.sock:/var/run/docker.sock") }) ); + let group_add = service + .get(serde_yaml::Value::String("group_add".to_string())) + .and_then(serde_yaml::Value::as_sequence) + .expect("controller service should join the router runtime group"); assert!( - service - .get(serde_yaml::Value::String("group_add".to_string())) - .is_none(), - "compose site controller should not override its supplemental groups; it runs as root to \ - preserve access to both the Docker socket and the router-control volume" + group_add + .iter() + .any(|value| value.as_str() == Some("65532")), + "compose site controller runs with dropped capabilities, so it needs the router runtime \ + group to access the router-control socket" ); assert!( extra_hosts @@ -9388,7 +9917,9 @@ fn kubernetes_site_controller_resources_are_injected_into_the_artifact() { fs::write( artifact_root.join("kustomization.yaml"), "resources:\n - 03-deployments/amber-site-controller.yaml\n - \ - 04-services/amber-site-controller.yaml\n - 05-networkpolicies/amber-router-netpol.yaml\n", + 04-services/amber-site-controller.yaml\n - \ + 05-networkpolicies/amber-router-netpol.yaml\n - \ + 05-networkpolicies/amber-site-controller-netpol.yaml\n", ) .expect("kustomization should write"); fs::write( @@ -9423,6 +9954,27 @@ spec: "#, ) .expect("router netpol should write"); + fs::write( + artifact_root.join("05-networkpolicies/amber-site-controller-netpol.yaml"), + r#" +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: amber-site-controller-netpol +spec: + egress: + - to: + - podSelector: + matchLabels: + amber.io/component: amber-router + ports: + - protocol: TCP + port: 24000 + - protocol: TCP + port: 24100 +"#, + ) + .expect("site controller netpol should write"); fs::write( site_state_root.join("site-controller-state.json"), "{\"schema\":\"amber.test\",\"version\":1}", @@ -9586,7 +10138,14 @@ spec: fs::read_to_string(artifact_root.join("01-configmaps/amber-site-controller-seed.yaml")) .expect("seed configmap should read"); assert!(seed.contains("site-controller-plan.json")); - assert!(seed.contains("artifact.tar.b64")); + assert!( + !seed.contains("artifact.tar.b64"), + "kubernetes controller bootstrap should seed ordinary files instead of a tarball" + ); + assert!( + deployment_raw.contains("cp -R /amber/seed/artifact/. /amber/site/artifact"), + "kubernetes controller bootstrap should copy seeded artifact files directly" + ); assert!(seed.contains(&format!("http://amber-site-controller:{controller_port}"))); assert!(deployment_raw.contains("EXISTING_ENV")); @@ -9600,6 +10159,11 @@ spec: .expect("router service should read"); assert!(router_service.contains("controller-route-37046")); assert!(router_service.contains("port: 37046")); + let controller_netpol = fs::read_to_string( + artifact_root.join("05-networkpolicies/amber-site-controller-netpol.yaml"), + ) + .expect("site controller netpol should read"); + assert!(controller_netpol.contains("37046")); } #[test] @@ -9688,6 +10252,81 @@ fn local_site_manager_state_uses_controller_plan_when_host_state_is_absent() { ); } +#[test] +fn site_execution_site_routes_vm_sites_to_their_direct_controller() { + let state = FrameworkControlState { + schema: CONTROL_STATE_SCHEMA.to_string(), + version: CONTROL_STATE_VERSION, + run_id: "test-run".to_string(), + base_scenario: ScenarioIr { + schema: amber_scenario::SCENARIO_IR_SCHEMA.to_string(), + version: amber_scenario::SCENARIO_IR_VERSION, + root: 0, + components: Vec::new(), + bindings: Vec::new(), + exports: Vec::new(), + manifest_catalog: BTreeMap::new(), + }, + run_links: Vec::new(), + placement: FrozenPlacementState { + offered_sites: BTreeMap::from([ + ( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "vm_local".to_string(), + SiteDefinition { + kind: SiteKind::Vm, + context: None, + controller_site: Some("direct_local".to_string()), + }, + ), + ]), + defaults: PlacementDefaults::default(), + standby_sites: Vec::new(), + initial_active_sites: vec!["direct_local".to_string(), "vm_local".to_string()], + dynamic_enabled_sites: vec!["direct_local".to_string(), "vm_local".to_string()], + control_only_sites: Vec::new(), + active_site_capabilities: BTreeMap::new(), + placement_components: BTreeMap::new(), + assignments: BTreeMap::new(), + }, + generation: 0, + next_child_id: 1, + next_tx_id: 0, + id_stride: 1, + next_component_id: 0, + capability_instances: BTreeMap::new(), + journal: Vec::new(), + dynamic_capability_signing_seed_b64: mesh_dynamic_caps::signing_seed_b64( + &mesh_dynamic_caps::signing_key_from_seed( + mesh_dynamic_caps::generate_dynamic_capability_signing_seed(), + ), + ), + next_dynamic_capability_grant_id: 0, + dynamic_capability_grants: BTreeMap::new(), + dynamic_capability_grant_authority_sites: BTreeMap::new(), + dynamic_capability_journal: Vec::new(), + live_children: Vec::new(), + pending_creates: Vec::new(), + pending_destroys: Vec::new(), + }; + + assert_eq!( + site_execution_site_from_state(&state, "direct_local").expect("direct site execution"), + "direct_local" + ); + assert_eq!( + site_execution_site_from_state(&state, "vm_local").expect("vm site execution"), + "direct_local" + ); +} + #[test] fn load_site_manager_state_prefers_local_controller_view_over_stale_host_state() { let temp = TempDir::new().expect("temp dir"); From a9d6d4379f4d6bad2e6b0c0550fae25ac2170d1c Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Sat, 2 May 2026 17:50:21 +0800 Subject: [PATCH 12/25] Fix more tests --- cli/src/direct_runtime/preview.rs | 78 +++++++----- cli/src/direct_runtime/process.rs | 12 ++ cli/src/direct_runtime/runtime.rs | 64 +++++----- cli/src/mixed_run.rs | 2 + cli/src/mixed_run/launch_bundle.rs | 38 ++++++ cli/src/mixed_run/supervisor.rs | 16 ++- cli/src/mixed_run/tests.rs | 21 ++++ cli/src/tests.rs | 68 +++++++++++ cli/tests/kvm_smoke.rs | 6 +- cli/tests/linux_vm.rs | 31 +++-- cli/tests/mixed_run/lifecycle.rs | 45 +++++-- cli/tests/mixed_run/support.rs | 95 ++++++++++++++- .../site-controller/src/default_runtime.rs | 111 +++++++++++++++++- .../src/default_runtime/child_runtime.rs | 2 + .../src/default_runtime/dynamic_routes.rs | 9 +- .../src/default_runtime/site_artifacts.rs | 2 +- runtime/site-controller/src/tests.rs | 11 +- 17 files changed, 502 insertions(+), 109 deletions(-) diff --git a/cli/src/direct_runtime/preview.rs b/cli/src/direct_runtime/preview.rs index dce44c03..aa350e23 100644 --- a/cli/src/direct_runtime/preview.rs +++ b/cli/src/direct_runtime/preview.rs @@ -128,6 +128,35 @@ pub(crate) fn component_program_spec( } } +#[cfg(target_os = "linux")] +pub(crate) fn direct_component_uses_isolated_network(component: &DirectComponentPlan) -> bool { + !matches!( + component.program.execution, + DirectProgramExecutionPlan::InternalSiteController + ) +} + +pub(crate) fn direct_component_sidecar_network(component: &DirectComponentPlan) -> ProcessNetwork { + #[cfg(target_os = "linux")] + { + if direct_component_uses_isolated_network(component) { + ProcessNetwork::Isolated + } else { + ProcessNetwork::Host + } + } + #[cfg(not(target_os = "linux"))] + { + let _ = component; + ProcessNetwork::Host + } +} + +#[cfg(target_os = "linux")] +pub(crate) fn direct_component_program_joins_sidecar(component: &DirectComponentPlan) -> bool { + direct_component_uses_isolated_network(component) +} + fn internal_site_controller_plan_path(runtime_root: &Path) -> PathBuf { let site_state_root = runtime_root.parent().unwrap_or(runtime_root); mixed_run::site_controller_plan_path(site_state_root) @@ -324,16 +353,7 @@ pub(crate) fn build_direct_site_launch_preview( bind_dirs: Vec::new(), bind_mounts: Vec::new(), hidden_paths: Vec::new(), - network: { - #[cfg(target_os = "linux")] - { - ProcessNetwork::Isolated - } - #[cfg(not(target_os = "linux"))] - { - ProcessNetwork::Host - } - }, + network: direct_component_sidecar_network(component), }, "sidecar", Some(component.moniker.as_str()), @@ -367,7 +387,7 @@ pub(crate) fn build_direct_site_launch_preview( spec, "program", Some(component.moniker.as_str()), - Some(direct_program_network_override()), + Some(direct_program_network_override(component)), resolved_process, )); } @@ -529,13 +549,18 @@ pub(crate) fn direct_process_network_label(network: ProcessNetwork) -> String { } } -pub(crate) fn direct_program_network_override() -> &'static str { +pub(crate) fn direct_program_network_override(component: &DirectComponentPlan) -> &'static str { #[cfg(target_os = "linux")] { - "join_component_sidecar" + if direct_component_program_joins_sidecar(component) { + "join_component_sidecar" + } else { + "host" + } } #[cfg(not(target_os = "linux"))] { + let _ = component; "host" } } @@ -645,13 +670,10 @@ pub(crate) async fn wait_for_component_local_targets( if ports.is_empty() { return Ok(()); } - let sidecar_pid = sidecar_pid.ok_or_else(|| { - miette::miette!( - "missing sidecar pid while waiting for component {} local targets", - component.moniker - ) - })?; - let namespace_join = prepare_linux_namespace_join(sidecar_pid)?; + let namespace_join = sidecar_pid + .map(prepare_linux_namespace_join) + .transpose()? + .flatten(); let amber_cli = env::current_exe() .into_diagnostic() .wrap_err("failed to locate current amber binary for direct local probe")?; @@ -1163,9 +1185,9 @@ pub(crate) async fn spawn_managed_process( ProcessSandbox::Sandboxed => sandbox.wrap_command(&spec)?, ProcessSandbox::Unsandboxed => { #[cfg(target_os = "linux")] - if !matches!(spec.network, ProcessNetwork::Host) { + if matches!(spec.network, ProcessNetwork::Isolated) { return Err(miette::miette!( - "unsandboxed direct processes must use host networking" + "unsandboxed direct processes cannot create isolated networking" )); } (spec.program.clone(), spec.args.clone()) @@ -1174,15 +1196,9 @@ pub(crate) async fn spawn_managed_process( #[cfg(target_os = "linux")] let mut args = args; #[cfg(target_os = "linux")] - let namespace_join = if matches!(spec.sandbox, ProcessSandbox::Sandboxed) - && matches!(sandbox, DirectSandbox::Bubblewrap { .. }) - { - match spec.network { - ProcessNetwork::Join(pid) => prepare_linux_namespace_join(pid)?, - _ => None, - } - } else { - None + let namespace_join = match spec.network { + ProcessNetwork::Join(pid) => prepare_linux_namespace_join(pid)?, + _ => None, }; #[cfg(target_os = "linux")] let pid_capture = if matches!(spec.sandbox, ProcessSandbox::Sandboxed) diff --git a/cli/src/direct_runtime/process.rs b/cli/src/direct_runtime/process.rs index c2324605..f8eb5d1c 100644 --- a/cli/src/direct_runtime/process.rs +++ b/cli/src/direct_runtime/process.rs @@ -588,6 +588,10 @@ pub(crate) async fn supervise_children( if child.wrapper_pid == child.managed_pid || !linux_pid_is_alive(child.managed_pid) { + eprintln!( + "direct runtime observed {} exit with status {}", + child.name, status + ); let exit_code = if status.success() { 0 } else { @@ -605,6 +609,10 @@ pub(crate) async fn supervise_children( #[cfg(target_os = "linux")] if child.wrapper.is_none() && !linux_pid_is_alive(child.managed_pid) { let status = synthetic_failure_exit_status(); + eprintln!( + "direct runtime observed {} exit without a wrapper status", + child.name + ); return Ok(( RuntimeExitReason::ChildExited { name: child.name.clone(), @@ -617,6 +625,10 @@ pub(crate) async fn supervise_children( if let Some(wrapper) = child.wrapper.as_mut() && let Some(status) = wrapper.try_wait().into_diagnostic()? { + eprintln!( + "direct runtime observed {} exit with status {}", + child.name, status + ); let exit_code = if status.success() { 0 } else { diff --git a/cli/src/direct_runtime/runtime.rs b/cli/src/direct_runtime/runtime.rs index e5b88459..15cabcc6 100644 --- a/cli/src/direct_runtime/runtime.rs +++ b/cli/src/direct_runtime/runtime.rs @@ -207,7 +207,7 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { "direct runtime requires `slirp4netns` on Linux for isolated component networking" ) })?; - let runtime_state = if args.skip_router || !existing_peer_ports_by_id.is_empty() { + let mut runtime_state = if args.skip_router || !existing_peer_ports_by_id.is_empty() { materialize_direct_runtime_with_existing( &plan_root, &runtime_root, @@ -448,21 +448,12 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { bind_dirs: vec![control_socket_dir], bind_mounts: Vec::new(), hidden_paths: Vec::new(), - network: { - #[cfg(target_os = "linux")] - { - ProcessNetwork::Isolated - } - #[cfg(not(target_os = "linux"))] - { - ProcessNetwork::Host - } - }, + network: direct_component_sidecar_network(component), }; let sidecar_pid = spawn_managed_process(spec, &mut sandbox, &mut children, &mut log_tasks).await?; #[cfg(target_os = "linux")] - { + if direct_component_uses_isolated_network(component) { let mesh_port = mesh_network .component_mesh_port_by_id .get(&component.id) @@ -504,13 +495,18 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { spec.hidden_paths.push(runtime_root.join("mesh")); #[cfg(target_os = "linux")] { - let pid = component_sidecar_pid_by_id - .get(component_id) - .copied() - .ok_or_else(|| { - miette::miette!("missing sidecar pid for component {}", component.moniker) - })?; - spec.network = ProcessNetwork::Join(pid); + if direct_component_program_joins_sidecar(component) { + let pid = component_sidecar_pid_by_id + .get(component_id) + .copied() + .ok_or_else(|| { + miette::miette!( + "missing sidecar pid for component {}", + component.moniker + ) + })?; + spec.network = ProcessNetwork::Join(pid); + } } let _ = spawn_managed_process(spec, &mut sandbox, &mut children, &mut log_tasks).await?; @@ -518,7 +514,11 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { wait_for_component_local_targets( component, &runtime_root, - component_sidecar_pid_by_id.get(component_id).copied(), + if direct_component_program_joins_sidecar(component) { + component_sidecar_pid_by_id.get(component_id).copied() + } else { + None + }, DIRECT_LOCAL_TARGET_READY_TIMEOUT, ) .await?; @@ -533,6 +533,7 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { } wait_for_direct_mesh_endpoints(&runtime_state, DIRECT_MESH_ENDPOINT_READY_TIMEOUT).await?; + runtime_state.ready = true; write_direct_runtime_state(&plan_root, &runtime_state)?; supervise_children(&mut children).await } @@ -832,6 +833,8 @@ pub(crate) fn direct_runtime_state_path(plan_root: &Path) -> PathBuf { #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub(crate) struct DirectRuntimeState { + #[serde(default)] + pub(crate) ready: bool, #[serde(default)] pub(crate) slot_ports_by_component: BTreeMap>, #[serde(default)] @@ -1181,7 +1184,10 @@ pub(crate) fn materialize_direct_runtime_with_existing( ) -> Result { let runtime_state_path = direct_runtime_state_path(plan_root); if existing.reuse_existing && runtime_state_path.is_file() { - return read_direct_runtime_state(&runtime_state_path); + let mut runtime_state = read_direct_runtime_state(&runtime_state_path)?; + runtime_state.ready = false; + write_direct_runtime_state(plan_root, &runtime_state)?; + return Ok(runtime_state); } if runtime_state_path.exists() { let _ = fs::remove_file(&runtime_state_path); @@ -1240,14 +1246,16 @@ pub(crate) fn configure_direct_mesh_network( })?; plan.component_mesh_port_by_id .insert(component.id, mesh_port); - config.mesh_listen = rewrite_mesh_listen_for_slirp_guest(config.mesh_listen); + if direct_component_uses_isolated_network(component) { + config.mesh_listen = rewrite_mesh_listen_for_slirp_guest(config.mesh_listen); - for route in &mut config.outbound { - route.peer_addr = rewrite_peer_addr_for_slirp_gateway(route.peer_addr.as_str()); - } - for route in &mut config.inbound { - if let InboundTarget::MeshForward { peer_addr, .. } = &mut route.target { - *peer_addr = rewrite_peer_addr_for_slirp_gateway(peer_addr.as_str()); + for route in &mut config.outbound { + route.peer_addr = rewrite_peer_addr_for_slirp_gateway(route.peer_addr.as_str()); + } + for route in &mut config.inbound { + if let InboundTarget::MeshForward { peer_addr, .. } = &mut route.target { + *peer_addr = rewrite_peer_addr_for_slirp_gateway(peer_addr.as_str()); + } } } diff --git a/cli/src/mixed_run.rs b/cli/src/mixed_run.rs index c96a2fea..e97b6778 100644 --- a/cli/src/mixed_run.rs +++ b/cli/src/mixed_run.rs @@ -633,6 +633,8 @@ pub(crate) struct ResolvedRunProxyTarget { #[derive(Debug, Deserialize)] struct DirectRuntimeStateView { + #[serde(default)] + ready: bool, #[serde(default)] router_mesh_port: Option, } diff --git a/cli/src/mixed_run/launch_bundle.rs b/cli/src/mixed_run/launch_bundle.rs index 671a3e8b..1dbccd93 100644 --- a/cli/src/mixed_run/launch_bundle.rs +++ b/cli/src/mixed_run/launch_bundle.rs @@ -79,6 +79,43 @@ fn site_controller_component_port(site_plan: &RunSitePlan) -> Result } } +pub(super) fn prepare_site_state_root(site_state_root: &Path, kind: SiteKind) -> Result<()> { + fs::create_dir_all(site_state_root) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to create site state directory {}", + site_state_root.display() + ) + })?; + + #[cfg(unix)] + if kind == SiteKind::Compose { + use std::os::unix::fs::PermissionsExt as _; + + let mut permissions = fs::metadata(site_state_root) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to read permissions for site state directory {}", + site_state_root.display() + ) + })? + .permissions(); + permissions.set_mode(0o777); + fs::set_permissions(site_state_root, permissions) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to make compose site state directory writable {}", + site_state_root.display() + ) + })?; + } + + Ok(()) +} + pub(crate) fn dry_run_run_plan( source_plan_path: Option<&Path>, run_plan: &RunPlan, @@ -159,6 +196,7 @@ pub(super) fn materialize_launch_bundle( let artifact_dir = materialize_site_artifacts(&sites_root, site_id, site_plan)?; amber_site_controller::set_site_artifact_mesh_identity_seed(&artifact_dir, run_id)?; let site_state_root = state_root.join(site_id); + prepare_site_state_root(&site_state_root, site_plan.site.kind)?; let controller = site_controller_component_port(site_plan)?.map(|controller_port| { let listen_addr = match site_plan.site.kind { SiteKind::Direct | SiteKind::Vm => { diff --git a/cli/src/mixed_run/supervisor.rs b/cli/src/mixed_run/supervisor.rs index 41b2f824..dc70107c 100644 --- a/cli/src/mixed_run/supervisor.rs +++ b/cli/src/mixed_run/supervisor.rs @@ -812,6 +812,9 @@ pub(super) async fn try_discover_direct_site( &direct_runtime_state_path(&artifact_dir), "direct runtime state", )?; + if !state.ready { + return Ok(None); + } let Some(router_mesh_port) = state.router_mesh_port else { return Ok(None); }; @@ -1893,9 +1896,20 @@ pub(super) fn spawn_runtime_process_with_executable( build: impl FnOnce(&mut Command), ) -> Result { let log_path = site_state_root.join(log_name); - let log = fs::File::create(&log_path) + let mut log = fs::OpenOptions::new() + .create(true) + .append(true) + .open(&log_path) .into_diagnostic() .wrap_err_with(|| format!("failed to create {}", log_path.display()))?; + writeln!(log, "\n--- starting runtime process ---") + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to write runtime start marker to {}", + log_path.display() + ) + })?; let log_err = log .try_clone() .into_diagnostic() diff --git a/cli/src/mixed_run/tests.rs b/cli/src/mixed_run/tests.rs index bc49db92..4438b9c5 100644 --- a/cli/src/mixed_run/tests.rs +++ b/cli/src/mixed_run/tests.rs @@ -36,6 +36,27 @@ fn site_state_paths_are_site_scoped() { ); } +#[cfg(unix)] +#[test] +fn compose_site_state_root_is_writable_by_container_uid_mappings() { + let temp = TempDir::new().expect("temp dir should create"); + let site_state_root = temp.path().join("state").join("compose-local"); + + launch_bundle::prepare_site_state_root(&site_state_root, SiteKind::Compose) + .expect("compose state root should prepare"); + + let mode = fs::metadata(&site_state_root) + .expect("state root metadata should read") + .permissions() + .mode() + & 0o777; + assert_eq!( + mode, 0o777, + "compose site controllers run in Docker and must be able to persist state through Linux \ + bind mounts even when the daemon remaps container uid 0" + ); +} + #[test] fn site_controller_image_override_uses_dev_tag() { let overrides = BTreeMap::from([( diff --git a/cli/src/tests.rs b/cli/src/tests.rs index b02fb860..078e6e41 100644 --- a/cli/src/tests.rs +++ b/cli/src/tests.rs @@ -220,6 +220,73 @@ fn proxy_telemetry_keeps_router_info_without_verbose_output() { ); } +#[cfg(target_os = "linux")] +#[test] +fn direct_runtime_only_isolates_non_controller_components() { + let controller = DirectComponentPlan { + id: 1, + moniker: "/__amber_internal_framework_component_controller/direct_local".to_string(), + log_name: "controller".to_string(), + source_dir: None, + depends_on: Vec::new(), + sidecar: amber_compiler::reporter::direct::DirectSidecarPlan { + log_name: "controller-sidecar".to_string(), + mesh_port: 0, + mesh_config_path: "mesh/components/controller/mesh-config.json".to_string(), + mesh_identity_path: "mesh/components/controller/mesh-identity.json".to_string(), + env_passthrough: Vec::new(), + }, + program: amber_compiler::reporter::direct::DirectProgramPlan { + log_name: "controller-program".to_string(), + work_dir: "work/components/controller".to_string(), + storage_mounts: Vec::new(), + execution: DirectProgramExecutionPlan::InternalSiteController, + }, + }; + let app = DirectComponentPlan { + id: 2, + moniker: "/app".to_string(), + log_name: "app".to_string(), + source_dir: Some("/workspace/scenarios/app".to_string()), + depends_on: Vec::new(), + sidecar: amber_compiler::reporter::direct::DirectSidecarPlan { + log_name: "app-sidecar".to_string(), + mesh_port: 0, + mesh_config_path: "mesh/components/app/mesh-config.json".to_string(), + mesh_identity_path: "mesh/components/app/mesh-identity.json".to_string(), + env_passthrough: Vec::new(), + }, + program: amber_compiler::reporter::direct::DirectProgramPlan { + log_name: "app-program".to_string(), + work_dir: "work/components/app".to_string(), + storage_mounts: Vec::new(), + execution: DirectProgramExecutionPlan::Direct { + entrypoint: vec!["/bin/true".to_string()], + env: BTreeMap::new(), + }, + }, + }; + + assert_eq!( + direct_component_sidecar_network(&controller), + ProcessNetwork::Host + ); + assert!(!direct_component_uses_isolated_network(&controller)); + assert!(!direct_component_program_joins_sidecar(&controller)); + assert_eq!(direct_program_network_override(&controller), "host"); + + assert_eq!( + direct_component_sidecar_network(&app), + ProcessNetwork::Isolated + ); + assert!(direct_component_uses_isolated_network(&app)); + assert!(direct_component_program_joins_sidecar(&app)); + assert_eq!( + direct_program_network_override(&app), + "join_component_sidecar" + ); +} + #[cfg(target_os = "linux")] #[test] fn component_program_read_only_mounts_resolve_parent_escape_paths() { @@ -310,6 +377,7 @@ fn build_runtime_template_context_uses_runtime_slot_ports() { dynamic_caps_port_by_component: BTreeMap::new(), component_mesh_port_by_id: BTreeMap::new(), router_mesh_port: None, + ready: false, }; let context = diff --git a/cli/tests/kvm_smoke.rs b/cli/tests/kvm_smoke.rs index 1ef95687..831245e3 100644 --- a/cli/tests/kvm_smoke.rs +++ b/cli/tests/kvm_smoke.rs @@ -14,7 +14,7 @@ use std::{ process::{Command, Stdio}, }; -use amber_images::{AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER}; +use amber_images::{AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER, AMBER_SITE_CONTROLLER}; use outputs_root_support::cli_test_outputs_root; use target_dir_support::cargo_target_dir; use workspace_root_support::workspace_root; @@ -88,6 +88,10 @@ fn ensure_internal_images(workspace_root: &Path) { AMBER_ROUTER.reference, &workspace_root.join("docker/amber-router/Dockerfile"), ); + build_internal_image_if_needed( + AMBER_SITE_CONTROLLER.reference, + &workspace_root.join("docker/amber-site-controller/Dockerfile"), + ); } fn build_internal_image_if_needed(tag: &str, dockerfile: &Path) { diff --git a/cli/tests/linux_vm.rs b/cli/tests/linux_vm.rs index 34a911eb..cb1a3c8a 100644 --- a/cli/tests/linux_vm.rs +++ b/cli/tests/linux_vm.rs @@ -807,8 +807,9 @@ impl LinuxVmHarness { AMBER_VM_SMOKE_BASE_IMAGE=\"$HOME/{image_filename}\"\nexport \ AMBER_MIXED_RUN_BASE_IMAGE=\"$HOME/{image_filename}\"\nexport \ CARGO_TARGET_DIR=\"$HOME/amber-target\"\nexport CARGO_TERM_QUIET=true\nexport \ - CARGO_TERM_PROGRESS_WHEN=never\n{extra_exports}log_dir=\"$HOME/\ - linux-vm-test-logs\"\nmkdir -p \ + CARGO_TERM_PROGRESS_WHEN=never\nexport CARGO_HTTP_MULTIPLEXING=false\nexport \ + CARGO_HTTP_TIMEOUT=120\nexport \ + CARGO_NET_RETRY=5\n{extra_exports}log_dir=\"$HOME/linux-vm-test-logs\"\nmkdir -p \ \"$log_dir\"\nstdout_log=\"$log_dir/{label}.stdout.log\"\nstderr_log=\"$log_dir/\ {label}.stderr.log\"\nif {command} >\"$stdout_log\" 2>\"$stderr_log\"; then\ncat \ \"$stdout_log\"\nelse\nstatus=$?\necho \"===== guest test panic lines \ @@ -840,19 +841,23 @@ impl LinuxVmHarness { then\ncontinue\nfi\necho \"===== state root: $state_root =====\"\nwhile IFS= read -r \ path; do\necho \"----- $path -----\"\nsed -n '1,260p' \"$path\"\ndone < <(find \ \"$state_root\" -type f \\( -name 'manager-state.json' -o -name \ - 'direct-runtime-state.json' -o -name 'vm-runtime-state.json' -o -name \ + 'direct-runtime-state.json' -o -name 'direct-runtime.json' -o -name \ + 'vm-runtime-state.json' -o -name 'vm-runtime.json' -o -name \ 'site-controller-state.json' -o -name 'site-controller-runtime-state.json' -o -name \ 'supervisor.log' -o -name 'port-forward.log' -o -name 'site.log' -o -name \ - 'site-controller.log' -o -name 'site-controller-plan.json' -o -name \ - 'site-controller-runtime-plan.json' \\) 2>/dev/null | \ - sort)\ndone\nruns_root=\"$(find {guest_workspace} -path '*/.amber-runs/runs/run-*' \ - -type d | sort | tail -n 1)\"\nif [ -n \"$runs_root\" ]; then\necho \"latest guest \ - run root: $runs_root\"\nfind \"$runs_root/state\" -maxdepth 2 -type f \\( -name \ - 'manager-state.json' -o -name 'supervisor.log' -o -name 'port-forward.log' -o -name \ - 'site.log' -o -name 'site-controller.log' -o -name 'site-controller-state.json' -o \ - -name 'site-controller-runtime-state.json' -o -name 'site-controller-plan.json' -o \ - -name 'site-controller-runtime-plan.json' \\) | sort | while IFS= read -r path; \ - do\necho \"----- $path -----\"\nsed -n '1,260p' \"$path\"\ndone\nfi\n" + 'outside-proxy.log' -o -name 'site-controller.log' -o -name \ + 'site-controller-plan.json' -o -name 'site-controller-runtime-plan.json' -o -name \ + 'mesh-config.json' \\) 2>/dev/null | sort)\ndone\nruns_root=\"$(find \ + {guest_workspace} -path '*/.amber-runs/runs/run-*' -type d | sort | tail -n 1)\"\nif \ + [ -n \"$runs_root\" ]; then\necho \"latest guest run root: $runs_root\"\nfind \ + \"$runs_root/state\" -maxdepth 2 -type f \\( -name 'manager-state.json' -o -name \ + 'direct-runtime.json' -o -name 'vm-runtime.json' -o -name 'supervisor.log' -o -name \ + 'port-forward.log' -o -name 'site.log' -o -name 'outside-proxy.log' -o -name \ + 'site-controller.log' -o -name 'site-controller-state.json' -o -name \ + 'site-controller-runtime-state.json' -o -name 'site-controller-plan.json' -o -name \ + 'site-controller-runtime-plan.json' -o -name 'mesh-config.json' \\) | sort | while \ + IFS= read -r path; do\necho \"----- $path -----\"\nsed -n '1,260p' \ + \"$path\"\ndone\nfi\n" ); self.ssh_output(&script) .ok() diff --git a/cli/tests/mixed_run/lifecycle.rs b/cli/tests/mixed_run/lifecycle.rs index 6f3f7681..83aec543 100644 --- a/cli/tests/mixed_run/lifecycle.rs +++ b/cli/tests/mixed_run/lifecycle.rs @@ -31,15 +31,38 @@ fn mixed_run_five_site_startup_state_and_teardown() { ["direct_a"] ]) ); + let assignments = run_plan["assignments"] + .as_object() + .expect("run plan assignments should be an object"); + let controller_prefix = "/__amber_internal_framework_component_controller/"; + let user_assignments = assignments + .iter() + .filter(|(component, _)| !component.starts_with(controller_prefix)) + .map(|(component, site)| (component.clone(), site.clone())) + .collect::>(); assert_eq!( - run_plan["assignments"], - json!({ - "/a": "direct_a", - "/b": "compose_b", - "/c": "kind_c", - "/d": "vm_d", - "/e": "compose_e" - }) + user_assignments, + serde_json::Map::from_iter([ + ("/a".to_string(), json!("direct_a")), + ("/b".to_string(), json!("compose_b")), + ("/c".to_string(), json!("kind_c")), + ("/d".to_string(), json!("vm_d")), + ("/e".to_string(), json!("compose_e")), + ]) + ); + let controller_assignments = assignments + .iter() + .filter(|(component, _)| component.starts_with(controller_prefix)) + .map(|(component, site)| (component.clone(), site.clone())) + .collect::>(); + assert_eq!( + controller_assignments, + serde_json::Map::from_iter([ + (format!("{controller_prefix}compose_b"), json!("compose_b")), + (format!("{controller_prefix}compose_e"), json!("compose_e")), + (format!("{controller_prefix}direct_a"), json!("direct_a")), + (format!("{controller_prefix}kind_c"), json!("kind_c")), + ]) ); assert_eq!( run.receipt["sites"] @@ -109,7 +132,7 @@ fn mixed_run_five_site_startup_state_and_teardown() { .as_object() .expect("direct external slots") .len(), - 2 + 5 ); assert_eq!( read_json( @@ -121,7 +144,7 @@ fn mixed_run_five_site_startup_state_and_teardown() { .as_object() .expect("compose_b external slots") .len(), - 2 + 5 ); assert_eq!( read_json( @@ -133,7 +156,7 @@ fn mixed_run_five_site_startup_state_and_teardown() { .as_object() .expect("kind external slots") .len(), - 1 + 4 ); assert_eq!( read_json( diff --git a/cli/tests/mixed_run/support.rs b/cli/tests/mixed_run/support.rs index b59ca102..5bba3fcf 100644 --- a/cli/tests/mixed_run/support.rs +++ b/cli/tests/mixed_run/support.rs @@ -612,12 +612,14 @@ pub(crate) fn wait_for_body( last_response = Some((status, body)); } if let Ok(Some(status)) = proxy.child.try_wait() { + let diagnostics = proxy_target_diagnostics(&proxy.output_dir); panic!( "amber proxy exited before http://127.0.0.1:{port}{path} became ready\nstatus: \ - {status}\noutput dir: {}\nlog ({}):\n{}", + {status}\noutput dir: {}\n{diagnostics}log ({}):\n{}", proxy.output_dir.display(), proxy.log_path.display(), - fs::read_to_string(&proxy.log_path).unwrap_or_default() + fs::read_to_string(&proxy.log_path).unwrap_or_default(), + diagnostics = diagnostics, ); } thread::sleep(Duration::from_millis(250)); @@ -625,15 +627,94 @@ pub(crate) fn wait_for_body( let last_response = last_response .map(|(status, body)| format!("last http response: {status}\n{body}\n")) .unwrap_or_else(|| "last http response: \n".to_string()); + let diagnostics = proxy_target_diagnostics(&proxy.output_dir); panic!( - "timed out waiting for http://127.0.0.1:{port}{path}\noutput dir: {}\n{last_response}log \ - ({}):\n{}", + "timed out waiting for http://127.0.0.1:{port}{path}\noutput dir: \ + {}\n{last_response}{diagnostics}log ({}):\n{}", proxy.output_dir.display(), proxy.log_path.display(), - fs::read_to_string(&proxy.log_path).unwrap_or_default() + fs::read_to_string(&proxy.log_path).unwrap_or_default(), + diagnostics = diagnostics, ); } +fn proxy_target_diagnostics(output_dir: &Path) -> String { + let Some(child_root) = output_dir.parent() else { + return String::new(); + }; + let mut files = Vec::new(); + collect_proxy_target_diagnostic_files(child_root, &mut files); + files.sort(); + files.dedup(); + if files.is_empty() { + return String::new(); + } + + let mut diagnostics = String::from("proxy target diagnostics:\n"); + for path in files { + diagnostics.push_str(&format!("----- {} -----\n", path.display())); + diagnostics.push_str(&read_diagnostic_file_excerpt(&path)); + diagnostics.push('\n'); + } + diagnostics +} + +fn collect_proxy_target_diagnostic_files(root: &Path, files: &mut Vec) { + let mut stack = vec![root.to_path_buf()]; + while let Some(path) = stack.pop() { + let Ok(metadata) = fs::symlink_metadata(&path) else { + continue; + }; + if metadata.is_file() { + if is_proxy_target_diagnostic_file(&path) { + files.push(path); + } + continue; + } + if !metadata.is_dir() { + continue; + } + let Ok(entries) = fs::read_dir(&path) else { + continue; + }; + for entry in entries.flatten() { + stack.push(entry.path()); + } + } +} + +fn is_proxy_target_diagnostic_file(path: &Path) -> bool { + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + return false; + }; + matches!( + name, + "direct-runtime.json" + | "vm-runtime.json" + | "direct-plan.json" + | "vm-plan.json" + | "mesh-provision-plan.json" + | "mesh-config.json" + | "site.log" + | "supervisor.log" + | "manager-state.json" + | "site-controller-runtime-state.json" + | "site-controller-state.json" + ) +} + +fn read_diagnostic_file_excerpt(path: &Path) -> String { + const MAX_BYTES: usize = 24 * 1024; + let Ok(mut bytes) = fs::read(path) else { + return "\n".to_string(); + }; + bytes.retain(|byte| *byte != 0); + if bytes.len() > MAX_BYTES { + bytes = bytes[bytes.len() - MAX_BYTES..].to_vec(); + } + String::from_utf8_lossy(&bytes).into_owned() +} + pub(crate) fn wait_for_path(proxy: &mut SpawnedProxy, port: u16, path: &str, timeout: Duration) { let _ = wait_for_body(proxy, port, path, timeout); } @@ -2756,9 +2837,11 @@ pub(crate) fn write_single_site_vm_fixture(root: &Path) -> ScenarioFixture { "schema": "amber.run.placement", "version": 1, "sites": { - "vm_local": { "kind": "vm" } + "direct_controller": { "kind": "direct" }, + "vm_local": { "kind": "vm", "controller_site": "direct_controller" } }, "defaults": { + "path": "direct_controller", "vm": "vm_local" } }), diff --git a/runtime/site-controller/src/default_runtime.rs b/runtime/site-controller/src/default_runtime.rs index 913ef75e..d193128f 100644 --- a/runtime/site-controller/src/default_runtime.rs +++ b/runtime/site-controller/src/default_runtime.rs @@ -227,6 +227,8 @@ struct DesiredExportPeer { #[derive(Clone, Debug, Default, Serialize, Deserialize)] struct DirectRuntimeState { + #[serde(default)] + ready: bool, #[serde(default)] slot_ports_by_component: BTreeMap>, #[serde(default)] @@ -1380,21 +1382,51 @@ fn filter_dynamic_mesh_provision_plan( write_json(&plan_path, &plan) } +#[derive(Clone, Copy)] +enum DetachedChildRuntimeReadiness { + DirectReady, + VmMaterialized, +} + +impl DetachedChildRuntimeReadiness { + fn description(self) -> &'static str { + match self { + Self::DirectReady => "direct child runtime state", + Self::VmMaterialized => "vm child runtime state", + } + } + + fn ready(self, state_path: &Path) -> Result { + match self { + Self::DirectReady => { + let state: DirectRuntimeState = read_json(state_path, self.description())?; + Ok(state.ready) + } + Self::VmMaterialized => { + let _: VmRuntimeState = read_json(state_path, self.description())?; + Ok(true) + } + } + } +} + async fn wait_for_detached_child_runtime_state( pid: u32, state_path: &Path, timeout: Duration, log_path: &Path, + readiness: DetachedChildRuntimeReadiness, ) -> Result<()> { let deadline = Instant::now() + timeout; while Instant::now() < deadline { - if state_path.is_file() { + if state_path.is_file() && readiness.ready(state_path)? { return Ok(()); } if !pid_is_alive(pid) { let log = fs::read_to_string(log_path).unwrap_or_default(); return Err(miette::miette!( - "dynamic child runtime exited before becoming ready\nlog ({}):\n{}", + "dynamic child runtime exited before {} became ready\nlog ({}):\n{}", + readiness.description(), log_path.display(), log )); @@ -1403,7 +1435,8 @@ async fn wait_for_detached_child_runtime_state( } let log = fs::read_to_string(log_path).unwrap_or_default(); Err(miette::miette!( - "timed out waiting for dynamic child runtime state {}\nlog ({}):\n{}", + "timed out waiting for dynamic {}\nstate ({}):\nlog ({}):\n{}", + readiness.description(), state_path.display(), log_path.display(), log @@ -1983,4 +2016,76 @@ mod tests { None => unsafe { env::remove_var("AMBER_VM_FORCE_TCG") }, } } + + #[tokio::test] + async fn detached_direct_child_runtime_state_waits_for_ready_flag() { + let temp = tempfile::tempdir().expect("temp dir should be created"); + let state_path = temp.path().join("direct-runtime.json"); + let log_path = temp.path().join("site.log"); + write_json( + &state_path, + &DirectRuntimeState { + ready: false, + ..Default::default() + }, + ) + .expect("direct runtime state should write"); + + let err = wait_for_detached_child_runtime_state( + std::process::id(), + &state_path, + Duration::from_millis(1), + &log_path, + DetachedChildRuntimeReadiness::DirectReady, + ) + .await + .expect_err("not-ready direct runtime state should time out"); + assert!( + err.to_string().contains("direct child runtime state"), + "error should identify the direct child runtime state: {err}" + ); + + write_json( + &state_path, + &DirectRuntimeState { + ready: true, + ..Default::default() + }, + ) + .expect("direct runtime state should write"); + wait_for_detached_child_runtime_state( + std::process::id(), + &state_path, + Duration::from_secs(1), + &log_path, + DetachedChildRuntimeReadiness::DirectReady, + ) + .await + .expect("ready direct runtime state should be accepted"); + } + + #[tokio::test] + async fn detached_vm_child_runtime_state_accepts_materialized_vm_state() { + let temp = tempfile::tempdir().expect("temp dir should be created"); + let state_path = temp.path().join("vm-runtime.json"); + let log_path = temp.path().join("site.log"); + write_json( + &state_path, + &VmRuntimeState { + router_mesh_port: Some(23000), + ..Default::default() + }, + ) + .expect("vm runtime state should write"); + + wait_for_detached_child_runtime_state( + std::process::id(), + &state_path, + Duration::from_secs(1), + &log_path, + DetachedChildRuntimeReadiness::VmMaterialized, + ) + .await + .expect("materialized VM runtime state should be accepted"); + } } diff --git a/runtime/site-controller/src/default_runtime/child_runtime.rs b/runtime/site-controller/src/default_runtime/child_runtime.rs index 70b92246..a8dc2d58 100644 --- a/runtime/site-controller/src/default_runtime/child_runtime.rs +++ b/runtime/site-controller/src/default_runtime/child_runtime.rs @@ -864,6 +864,7 @@ pub(super) async fn site_controller_runtime_publish_child( &direct_runtime_state_path(Path::new(&child.artifact_root)), site_ready_timeout_for_kind(SiteKind::Direct), &child_root.join("site.log"), + DetachedChildRuntimeReadiness::DirectReady, ) .await?; let mut live_children = published_children.clone(); @@ -971,6 +972,7 @@ pub(super) async fn site_controller_runtime_publish_child( .join("vm-runtime.json"), vm_endpoint_forward_ready_timeout(), &child_root.join("site.log"), + DetachedChildRuntimeReadiness::VmMaterialized, ) .await?; wait_for_detached_vm_child_endpoints_ready( diff --git a/runtime/site-controller/src/default_runtime/dynamic_routes.rs b/runtime/site-controller/src/default_runtime/dynamic_routes.rs index 026c299b..3e17810b 100644 --- a/runtime/site-controller/src/default_runtime/dynamic_routes.rs +++ b/runtime/site-controller/src/default_runtime/dynamic_routes.rs @@ -2056,13 +2056,8 @@ mod direct_input_tests { ref peer_addr, ref route_id, ref capability, - } if peer_id == "/site/local/controller" - && peer_addr - == if cfg!(target_os = "linux") { - "10.0.2.2:23001" - } else { - "127.0.0.1:23001" - } + } if peer_id == "/site/local/controller" + && peer_addr == "127.0.0.1:23001" && route_id == &amber_mesh::component_route_id( "/site/local/controller", diff --git a/runtime/site-controller/src/default_runtime/site_artifacts.rs b/runtime/site-controller/src/default_runtime/site_artifacts.rs index 84ba179e..34fc60d6 100644 --- a/runtime/site-controller/src/default_runtime/site_artifacts.rs +++ b/runtime/site-controller/src/default_runtime/site_artifacts.rs @@ -98,7 +98,7 @@ fn local_site_controller_inbound_target( Some(EmbeddedMeshPlanKind::Kubernetes) => { kubernetes_component_mesh_peer_addr(artifact_root, &peer_id, &target.output, mesh_port)? } - None => local_component_mesh_peer_addr(mesh_port), + None => local_router_mesh_peer_addr(mesh_port), }; Ok(InboundTarget::MeshForward { peer_addr, diff --git a/runtime/site-controller/src/tests.rs b/runtime/site-controller/src/tests.rs index f2bc7fe3..36940a88 100644 --- a/runtime/site-controller/src/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -31,6 +31,8 @@ use crate::{ #[derive(Clone, Debug, Default, Serialize, Deserialize)] struct DirectRuntimeState { + #[serde(default)] + ready: bool, #[serde(default)] slot_ports_by_component: BTreeMap>, #[serde(default)] @@ -3788,13 +3790,8 @@ fn inject_site_controller_peer_router_routes_records_peer_router_identities() { ref peer_addr, ref route_id, ref capability, - } if peer_id == "/site/local/controller" - && peer_addr - == if cfg!(target_os = "linux") { - "10.0.2.2:23001" - } else { - "127.0.0.1:23001" - } + } if peer_id == "/site/local/controller" + && peer_addr == "127.0.0.1:23001" && route_id == &amber_mesh::component_route_id( "/site/local/controller", From 5fbe7a21269b4a26a64c040bea92caefa00ef974 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Sat, 2 May 2026 21:01:51 +0800 Subject: [PATCH 13/25] Fix more tests --- .github/workflows/ci.yaml | 44 ++++++++++- cli/src/mixed_run/supervisor.rs | 28 +++++++ cli/tests/kvm_smoke.rs | 46 ++++++----- .../src/targets/mesh/kubernetes/tests/mod.rs | 77 ++++++++++++++++++- .../targets/mesh/kubernetes/tests/storage.rs | 10 +++ 5 files changed, 181 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8a2364af..effbdf26 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -183,7 +183,12 @@ jobs: kvm-smoke-tests: name: KVM Smoke Tests runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 30 + needs: docker-build + if: ${{ always() && (github.event_name != 'pull_request' || needs.docker-build.result == 'success') }} + permissions: + contents: read + packages: read env: SCCACHE_GHA_ENABLED: "true" RUSTC_WRAPPER: "sccache" @@ -191,8 +196,43 @@ jobs: - uses: actions/checkout@v5 - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@v0.0.9 + - name: Compute image registries + if: github.event_name == 'pull_request' + shell: bash + run: | + set -euo pipefail + owner_lc="$(printf '%s' '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" + ci_registry="ghcr.io/${owner_lc}" + code_registry="$(jq -r '.registry' docker/images.json)" + if [ -z "$code_registry" ] || [ "$code_registry" = "null" ]; then + echo "docker/images.json registry is missing" >&2 + exit 1 + fi + code_registry="${code_registry%/}" + echo "CI_IMAGE_REGISTRY=${ci_registry}" >> "$GITHUB_ENV" + echo "CODE_IMAGE_REGISTRY=${code_registry}" >> "$GITHUB_ENV" + - uses: docker/login-action@v3 + if: github.event_name == 'pull_request' + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Pull and retag images for KVM smoke test + if: github.event_name == 'pull_request' + shell: bash + run: | + SOURCE_IMAGE_REGISTRY="$CI_IMAGE_REGISTRY" \ + TARGET_IMAGE_REGISTRY="$CODE_IMAGE_REGISTRY" \ + SOURCE_IMAGE_TAG="$GITHUB_SHA" \ + ./.github/scripts/prepare_prebuilt_images.sh - name: Run KVM smoke test - run: cargo test -p amber-cli --test kvm_smoke -- --ignored --nocapture --test-threads=1 + shell: bash + run: | + set -euo pipefail + if [ "${{ github.event_name }}" = "pull_request" ]; then + export AMBER_TEST_USE_PREBUILT_IMAGES=1 + fi + cargo test -p amber-cli --test kvm_smoke -- --ignored --nocapture --test-threads=1 kubernetes-tests: name: Kubernetes Reporter Tests diff --git a/cli/src/mixed_run/supervisor.rs b/cli/src/mixed_run/supervisor.rs index dc70107c..c3bbbc75 100644 --- a/cli/src/mixed_run/supervisor.rs +++ b/cli/src/mixed_run/supervisor.rs @@ -176,6 +176,12 @@ pub(super) fn launch_env( env.entry(amber_images::DEV_IMAGE_TAGS_ENV.to_string()) .or_insert_with(|| overrides.to_string()); } + if let Some(force_tcg) = std::env::var_os("AMBER_VM_FORCE_TCG") + && let Some(force_tcg) = force_tcg.to_str() + { + env.entry("AMBER_VM_FORCE_TCG".to_string()) + .or_insert_with(|| force_tcg.to_string()); + } env.insert(SCENARIO_RUN_ID_ENV.to_string(), run_id.to_string()); env.insert(SCENARIO_SCOPE_ENV.to_string(), mesh_scope.to_string()); if let Some(endpoint) = observability_endpoint { @@ -2256,6 +2262,28 @@ mod tests { } } + #[test] + fn launch_env_includes_vm_tcg_override() { + let previous = std::env::var_os("AMBER_VM_FORCE_TCG"); + unsafe { + std::env::set_var("AMBER_VM_FORCE_TCG", "1"); + } + let env = launch_env( + "run-test", + "scope", + SiteKind::Kubernetes, + &BTreeMap::new(), + &BTreeMap::new(), + None, + ) + .expect("launch env should build"); + assert_eq!(env.get("AMBER_VM_FORCE_TCG").map(String::as_str), Some("1")); + match previous { + Some(value) => unsafe { std::env::set_var("AMBER_VM_FORCE_TCG", value) }, + None => unsafe { std::env::remove_var("AMBER_VM_FORCE_TCG") }, + } + } + #[test] fn cross_site_container_consumers_resolve_links_as_mesh_router_urls() { let temp = tempfile::tempdir().expect("tempdir should create"); diff --git a/cli/tests/kvm_smoke.rs b/cli/tests/kvm_smoke.rs index 831245e3..9f9281fd 100644 --- a/cli/tests/kvm_smoke.rs +++ b/cli/tests/kvm_smoke.rs @@ -8,7 +8,7 @@ mod target_dir_support; mod workspace_root_support; use std::{ - fs, + env, fs, os::unix::fs::MetadataExt, path::{Path, PathBuf}, process::{Command, Stdio}, @@ -76,28 +76,38 @@ fn build_checker_image(workspace_root: &Path) { } fn ensure_internal_images(workspace_root: &Path) { - build_internal_image_if_needed( - AMBER_HELPER.reference, - &workspace_root.join("docker/amber-helper/Dockerfile"), - ); - build_internal_image_if_needed( - AMBER_PROVISIONER.reference, - &workspace_root.join("docker/amber-provisioner/Dockerfile"), - ); - build_internal_image_if_needed( - AMBER_ROUTER.reference, - &workspace_root.join("docker/amber-router/Dockerfile"), - ); - build_internal_image_if_needed( - AMBER_SITE_CONTROLLER.reference, - &workspace_root.join("docker/amber-site-controller/Dockerfile"), - ); + for (tag, dockerfile) in [ + ( + AMBER_HELPER.reference, + workspace_root.join("docker/amber-helper/Dockerfile"), + ), + ( + AMBER_PROVISIONER.reference, + workspace_root.join("docker/amber-provisioner/Dockerfile"), + ), + ( + AMBER_ROUTER.reference, + workspace_root.join("docker/amber-router/Dockerfile"), + ), + ( + AMBER_SITE_CONTROLLER.reference, + workspace_root.join("docker/amber-site-controller/Dockerfile"), + ), + ] { + ensure_internal_image(tag, &dockerfile); + } } -fn build_internal_image_if_needed(tag: &str, dockerfile: &Path) { +fn ensure_internal_image(tag: &str, dockerfile: &Path) { if docker_image_exists(tag) { return; } + if env::var_os("AMBER_TEST_USE_PREBUILT_IMAGES").is_some() { + panic!( + "AMBER_TEST_USE_PREBUILT_IMAGES is set but {tag} is not available locally. Pull and \ + retag CI images before running the KVM smoke test." + ); + } progress(format!("building internal image {tag}")); let output = Command::new("docker") diff --git a/compiler/src/targets/mesh/kubernetes/tests/mod.rs b/compiler/src/targets/mesh/kubernetes/tests/mod.rs index 9165219e..52406787 100644 --- a/compiler/src/targets/mesh/kubernetes/tests/mod.rs +++ b/compiler/src/targets/mesh/kubernetes/tests/mod.rs @@ -403,10 +403,22 @@ fn write_kubernetes_smoke_fixture(root: &Path) -> PathBuf { "\ mkdir content\n\ cd content\n\ - wget '${slots.server.url}/runtime_secret.txt'\n\ - wget '${slots.server.url}/runtime_config.txt'\n\ - wget '${slots.server.url}/static_secret.txt'\n\ - wget '${slots.server.url}/static_config.txt'\n\ + fetch() {\n\ + path=\"$1\"\n\ + attempts=0\n\ + until wget -O \"$path\" '${slots.server.url}/'\"$path\"; do\n\ + rm -f \"$path\"\n\ + attempts=$((attempts + 1))\n\ + if [ \"$attempts\" -ge 60 ]; then\n\ + exit 1\n\ + fi\n\ + sleep 1\n\ + done\n\ + }\n\ + fetch runtime_secret.txt\n\ + fetch runtime_config.txt\n\ + fetch static_secret.txt\n\ + fetch static_config.txt\n\ httpd -f -p 8080\n\ ", ], @@ -1018,6 +1030,63 @@ fn kubectl_logs(namespace: &str, pod: &str, kubeconfig: &Path) -> String { } } +fn wait_for_pod_http( + namespace: &str, + pod: &str, + container: &str, + url: &str, + kubeconfig: &Path, + timeout: Duration, +) -> String { + let deadline = Instant::now() + timeout; + let mut last_status = String::new(); + let mut last_stdout = String::new(); + let mut last_stderr = String::new(); + + loop { + let output = kubectl_cmd(kubeconfig) + .arg("exec") + .arg("-n") + .arg(namespace) + .arg(pod) + .arg("-c") + .arg(container) + .arg("--") + .arg("wget") + .arg("-q") + .arg("-O") + .arg("-") + .arg(url) + .output(); + match output { + Ok(output) if output.status.success() => { + return String::from_utf8_lossy(&output.stdout).trim().to_string(); + } + Ok(output) => { + last_status = output.status.to_string(); + last_stdout = String::from_utf8_lossy(&output.stdout).to_string(); + last_stderr = String::from_utf8_lossy(&output.stderr).to_string(); + } + Err(err) => { + last_status = format!("failed to run kubectl exec: {err}"); + last_stdout.clear(); + last_stderr.clear(); + } + } + + if Instant::now() >= deadline { + let pod_logs = kubectl_logs(namespace, pod, kubeconfig); + panic!( + "pod HTTP endpoint {url} did not become ready in container {container}\nlast \ + status: {last_status}\nlast stdout:\n{last_stdout}\nlast \ + stderr:\n{last_stderr}\npod logs:\n{pod_logs}" + ); + } + + thread::sleep(Duration::from_secs(1)); + } +} + fn fetch( url: &str, port_forward: &mut PortForwardGuard, diff --git a/compiler/src/targets/mesh/kubernetes/tests/storage.rs b/compiler/src/targets/mesh/kubernetes/tests/storage.rs index e1b0181c..c08b03b2 100644 --- a/compiler/src/targets/mesh/kubernetes/tests/storage.rs +++ b/compiler/src/targets/mesh/kubernetes/tests/storage.rs @@ -372,6 +372,16 @@ fn kubernetes_smoke_config_roundtrip() { .arg(&client_pod); checked_status(&mut cmd, "kubectl wait for client pod"); + let client_runtime_secret = wait_for_pod_http( + &namespace, + &client_pod, + "main", + "http://127.0.0.1:8080/runtime_secret.txt", + &kubeconfig, + Duration::from_secs(120), + ); + assert_eq!(client_runtime_secret, "test-secret-value"); + let port_forward_log = dir.path().join("port-forward.log"); let mut port_forward = PortForwardGuard::new(&namespace, &client_pod, &port_forward_log, &kubeconfig); From 33c75a5696dc8ab8dc2a467e92f3815be9ee146c Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Sat, 2 May 2026 23:14:45 +0800 Subject: [PATCH 14/25] Fix more tests --- cli/src/direct_runtime/runtime.rs | 35 +++++----- cli/src/mixed_run.rs | 4 ++ cli/src/mixed_run/supervisor.rs | 2 +- cli/src/mixed_run/tests.rs | 13 ++++ cli/src/vm_runtime/state.rs | 35 +++++----- .../site-controller/src/default_runtime.rs | 45 ++++++++++++- .../src/default_runtime/child_runtime.rs | 7 +- .../src/default_runtime/compose_controller.rs | 1 + .../src/default_runtime/dynamic_routes.rs | 4 ++ .../default_runtime/kubernetes_controller.rs | 1 + .../src/default_runtime/site_artifacts.rs | 1 + .../default_runtime/site_runtime_support.rs | 64 +++++++++++++++---- runtime/site-controller/src/lib.rs | 2 +- runtime/site-controller/src/runtime_api.rs | 3 + runtime/site-controller/src/state.rs | 5 ++ runtime/site-controller/src/tests.rs | 1 + 16 files changed, 171 insertions(+), 52 deletions(-) diff --git a/cli/src/direct_runtime/runtime.rs b/cli/src/direct_runtime/runtime.rs index 15cabcc6..48b2e9f9 100644 --- a/cli/src/direct_runtime/runtime.rs +++ b/cli/src/direct_runtime/runtime.rs @@ -886,11 +886,19 @@ pub(crate) fn assign_direct_runtime_ports_with_existing( } mesh_port_by_peer_id.insert(peer_id.clone(), *port); } + if let Some(port) = fixed_router_mesh_port + && !reserved.insert(port) + { + return Err(miette::miette!( + "router mesh port {} conflicts with an existing direct runtime port", + port + )); + } for component in &direct_plan.components { let path = runtime_root.join(&component.sidecar.mesh_config_path); let mut config = read_mesh_config_public(path.as_path())?; - let mesh_port = allocate_direct_runtime_port(&mut reserved, None)?; + let mesh_port = allocate_direct_runtime_port(&mut reserved)?; if mesh_port_by_peer_id .insert(config.identity.id.clone(), mesh_port) .is_some() @@ -902,7 +910,7 @@ pub(crate) fn assign_direct_runtime_ports_with_existing( } config.mesh_listen = SocketAddr::new(config.mesh_listen.ip(), mesh_port); if let Some(dynamic_caps_listen) = config.dynamic_caps_listen.as_mut() { - let port = allocate_direct_runtime_port(&mut reserved, None)?; + let port = allocate_direct_runtime_port(&mut reserved)?; *dynamic_caps_listen = SocketAddr::new(dynamic_caps_listen.ip(), port); state .dynamic_caps_port_by_component @@ -912,7 +920,7 @@ pub(crate) fn assign_direct_runtime_ports_with_existing( let mut slot_route_ports: BTreeMap> = BTreeMap::new(); for route in &mut config.outbound { let authored_port = route.listen_port; - let port = allocate_direct_runtime_port(&mut reserved, None)?; + let port = allocate_direct_runtime_port(&mut reserved)?; route.listen_port = port; slot_route_ports .entry(route.slot.clone()) @@ -958,7 +966,10 @@ pub(crate) fn assign_direct_runtime_ports_with_existing( let mut router_config = if let Some(router) = direct_plan.router.as_ref() { let path = runtime_root.join(&router.mesh_config_path); let mut config = read_mesh_config_public(path.as_path())?; - let mesh_port = allocate_direct_runtime_port(&mut reserved, fixed_router_mesh_port)?; + let mesh_port = match fixed_router_mesh_port { + Some(port) => port, + None => allocate_direct_runtime_port(&mut reserved)?, + }; if let Some(existing) = mesh_port_by_peer_id.insert(config.identity.id.clone(), mesh_port) && existing != mesh_port { @@ -1009,19 +1020,7 @@ pub(crate) fn cross_site_router_mesh_bind_ip( } } -pub(crate) fn allocate_direct_runtime_port( - reserved: &mut BTreeSet, - preferred: Option, -) -> Result { - if let Some(preferred) = preferred { - if reserved.insert(preferred) { - return Ok(preferred); - } - return Err(miette::miette!( - "runtime port {} was requested twice in one direct runtime", - preferred - )); - } +pub(crate) fn allocate_direct_runtime_port(reserved: &mut BTreeSet) -> Result { for _ in 0..256 { let port = amber_site_controller::reserve_loopback_port()?; if reserved.insert(port) { @@ -1434,7 +1433,7 @@ mod tests { let mut reserved = BTreeSet::new(); for _ in 0..32 { - let port = allocate_direct_runtime_port(&mut reserved, None) + let port = allocate_direct_runtime_port(&mut reserved) .expect("direct runtime port allocation should succeed"); assert_ne!( port, reserved_port, diff --git a/cli/src/mixed_run.rs b/cli/src/mixed_run.rs index e97b6778..46c714ea 100644 --- a/cli/src/mixed_run.rs +++ b/cli/src/mixed_run.rs @@ -1114,6 +1114,10 @@ pub(crate) fn reserve_loopback_port() -> Result { amber_site_controller::reserve_loopback_port() } +pub(crate) fn reserve_host_port() -> Result { + amber_site_controller::reserve_host_port() +} + pub(crate) fn new_run_id() -> String { let millis = SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/cli/src/mixed_run/supervisor.rs b/cli/src/mixed_run/supervisor.rs index c3bbbc75..e546a4d5 100644 --- a/cli/src/mixed_run/supervisor.rs +++ b/cli/src/mixed_run/supervisor.rs @@ -244,7 +244,7 @@ pub(super) fn build_supervisor_plan( input.site_plan.site.kind, SiteKind::Direct | SiteKind::Vm | SiteKind::Compose ) { - Some(reserve_loopback_port()?) + Some(reserve_host_port()?) } else { None }, diff --git a/cli/src/mixed_run/tests.rs b/cli/src/mixed_run/tests.rs index 4438b9c5..ddbea877 100644 --- a/cli/src/mixed_run/tests.rs +++ b/cli/src/mixed_run/tests.rs @@ -499,6 +499,19 @@ fn reserve_loopback_port_shares_allocator_with_site_controller_runtime() { } } +#[test] +fn reserve_host_port_shares_allocator_with_loopback_reservations() { + let host_port = reserve_host_port().expect("host port reservation should succeed"); + for _ in 0..32 { + let loopback_port = + reserve_loopback_port().expect("loopback port reservation should succeed"); + assert_ne!( + loopback_port, host_port, + "host and loopback reservations must use the same shared pool", + ); + } +} + #[cfg(unix)] #[test] fn stop_kubernetes_namespace_force_deletes_stuck_pods_before_retrying() { diff --git a/cli/src/vm_runtime/state.rs b/cli/src/vm_runtime/state.rs index 90d04d42..ad18e209 100644 --- a/cli/src/vm_runtime/state.rs +++ b/cli/src/vm_runtime/state.rs @@ -228,11 +228,19 @@ pub(crate) fn assign_vm_runtime_ports_with_existing( } mesh_port_by_peer_id.insert(peer_id.clone(), *port); } + if let Some(port) = fixed_router_mesh_port + && !reserved.insert(port) + { + return Err(miette::miette!( + "router mesh port {} conflicts with an existing vm runtime port", + port + )); + } for component in &vm_plan.components { let path = runtime_root.join(&component.mesh_config_path); let mut config = read_mesh_config_public(&path)?; - let mesh_port = allocate_runtime_port(&mut reserved, None)?; + let mesh_port = allocate_runtime_port(&mut reserved)?; if mesh_port_by_peer_id .insert(config.identity.id.clone(), mesh_port) .is_some() @@ -247,7 +255,7 @@ pub(crate) fn assign_vm_runtime_ports_with_existing( let mut route_guest_host_pairs = BTreeMap::>::new(); for route in &mut config.outbound { let guest_port = route.listen_port; - let host_port = allocate_runtime_port(&mut reserved, None)?; + let host_port = allocate_runtime_port(&mut reserved)?; route.listen_port = host_port; route_guest_host_pairs .entry(route.slot.clone()) @@ -289,7 +297,7 @@ pub(crate) fn assign_vm_runtime_ports_with_existing( let host_port = if let Some(existing) = endpoint_forwards.get(&guest_port) { *existing } else { - let host_port = allocate_runtime_port(&mut reserved, None)?; + let host_port = allocate_runtime_port(&mut reserved)?; endpoint_forwards.insert(guest_port, host_port); host_port }; @@ -324,7 +332,10 @@ pub(crate) fn assign_vm_runtime_ports_with_existing( let mut router_config = if let Some(router) = vm_plan.router.as_ref() { let path = runtime_root.join(&router.mesh_config_path); let mut config = read_mesh_config_public(&path)?; - let mesh_port = allocate_runtime_port(&mut reserved, fixed_router_mesh_port)?; + let mesh_port = match fixed_router_mesh_port { + Some(port) => port, + None => allocate_runtime_port(&mut reserved)?, + }; if let Some(existing) = mesh_port_by_peer_id.insert(config.identity.id.clone(), mesh_port) && existing != mesh_port { @@ -365,19 +376,7 @@ pub(crate) fn assign_vm_runtime_ports_with_existing( }) } -pub(crate) fn allocate_runtime_port( - reserved: &mut BTreeSet, - preferred: Option, -) -> Result { - if let Some(preferred) = preferred { - if reserved.insert(preferred) { - return Ok(preferred); - } - return Err(miette::miette!( - "runtime port {} was requested twice in one vm runtime", - preferred - )); - } +pub(crate) fn allocate_runtime_port(reserved: &mut BTreeSet) -> Result { for _ in 0..256 { let port = amber_site_controller::reserve_loopback_port()?; if reserved.insert(port) { @@ -1057,7 +1056,7 @@ mod tests { let mut reserved = BTreeSet::new(); for _ in 0..32 { - let port = allocate_runtime_port(&mut reserved, None) + let port = allocate_runtime_port(&mut reserved) .expect("vm runtime port allocation should succeed"); assert_ne!( port, reserved_port, diff --git a/runtime/site-controller/src/default_runtime.rs b/runtime/site-controller/src/default_runtime.rs index d193128f..c41aa1af 100644 --- a/runtime/site-controller/src/default_runtime.rs +++ b/runtime/site-controller/src/default_runtime.rs @@ -62,7 +62,7 @@ pub use self::{ }, site_runtime_support::{ host_service_bind_addr_for_consumer, observability_endpoint_for_site, - prepare_kubernetes_artifact_namespace, reserve_loopback_port, + prepare_kubernetes_artifact_namespace, reserve_host_port, reserve_loopback_port, router_mesh_addr_for_consumer, site_controller_peer_router_url, walk_files, }, }; @@ -375,6 +375,14 @@ pub fn vm_endpoint_forward_ready_timeout() -> Duration { } } +fn vm_endpoint_forward_ready_timeout_for_runtime_plan( + plan: &SiteControllerRuntimePlan, +) -> Duration { + plan.vm_endpoint_forward_ready_timeout_secs + .map(Duration::from_secs) + .unwrap_or_else(vm_endpoint_forward_ready_timeout) +} + fn required_existing_mesh_peer_identities( plan: &MeshProvisionPlan, available_peer_identities_by_id: &BTreeMap, @@ -1901,6 +1909,7 @@ mod tests { kubernetes_namespace: Some("ns".to_string()), context: None, observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, launch_env: BTreeMap::new(), }; let stale_manager_state = SiteManagerState { @@ -1961,6 +1970,7 @@ mod tests { kubernetes_namespace: Some("ns".to_string()), context: None, observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, launch_env: BTreeMap::new(), }; let manager_state = SiteManagerState { @@ -2017,6 +2027,39 @@ mod tests { } } + #[test] + fn runtime_plan_vm_ready_timeout_uses_explicit_budget() { + let plan = SiteControllerRuntimePlan { + schema: "amber.run.site_controller_runtime_plan".to_string(), + version: 1, + run_id: "run".to_string(), + mesh_scope: "scope".to_string(), + run_root: "/tmp/run".to_string(), + site_id: "vm_local".to_string(), + kind: SiteKind::Vm, + router_identity_id: "/site/vm_local/router".to_string(), + local_router_control: None, + artifact_dir: "/tmp/artifact".to_string(), + site_state_root: "/tmp/state".to_string(), + listen_addr: SocketAddr::from(([127, 0, 0, 1], 4100)), + storage_root: None, + runtime_root: None, + router_mesh_port: Some(24000), + compose_project: None, + kubernetes_namespace: None, + context: None, + observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: Some(721), + launch_env: BTreeMap::new(), + }; + + assert_eq!( + vm_endpoint_forward_ready_timeout_for_runtime_plan(&plan), + Duration::from_secs(721), + "dynamic VM child publication should consume the budget written into the runtime plan", + ); + } + #[tokio::test] async fn detached_direct_child_runtime_state_waits_for_ready_flag() { let temp = tempfile::tempdir().expect("temp dir should be created"); diff --git a/runtime/site-controller/src/default_runtime/child_runtime.rs b/runtime/site-controller/src/default_runtime/child_runtime.rs index a8dc2d58..0dcc2b44 100644 --- a/runtime/site-controller/src/default_runtime/child_runtime.rs +++ b/runtime/site-controller/src/default_runtime/child_runtime.rs @@ -965,12 +965,13 @@ pub(super) async fn site_controller_runtime_publish_child( record.process_pid = Some(process.id()); write_json(&app.state_path, &*state)?; } + let vm_ready_timeout = vm_endpoint_forward_ready_timeout_for_runtime_plan(&app.plan); wait_for_detached_child_runtime_state( process.id(), &Path::new(&child.artifact_root) .join(".amber") .join("vm-runtime.json"), - vm_endpoint_forward_ready_timeout(), + vm_ready_timeout, &child_root.join("site.log"), DetachedChildRuntimeReadiness::VmMaterialized, ) @@ -979,7 +980,7 @@ pub(super) async fn site_controller_runtime_publish_child( process.id(), Path::new(&child.artifact_root), &runtime_root, - vm_endpoint_forward_ready_timeout(), + vm_ready_timeout, &child_root.join("site.log"), )?; let mut live_children = published_children.clone(); @@ -1450,6 +1451,7 @@ mod tests { kubernetes_namespace: None, context: None, observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, launch_env: BTreeMap::new(), }; @@ -1591,6 +1593,7 @@ mod tests { kubernetes_namespace: None, context: None, observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, launch_env: BTreeMap::new(), }; diff --git a/runtime/site-controller/src/default_runtime/compose_controller.rs b/runtime/site-controller/src/default_runtime/compose_controller.rs index b8201744..5c61f2a1 100644 --- a/runtime/site-controller/src/default_runtime/compose_controller.rs +++ b/runtime/site-controller/src/default_runtime/compose_controller.rs @@ -264,6 +264,7 @@ mod tests { kubernetes_namespace: None, context: None, observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, launch_env: BTreeMap::from([( "AMBER_DEV_IMAGE_TAGS".to_string(), "router=dev-tag,helper=dev-tag".to_string(), diff --git a/runtime/site-controller/src/default_runtime/dynamic_routes.rs b/runtime/site-controller/src/default_runtime/dynamic_routes.rs index 3e17810b..9cbf9246 100644 --- a/runtime/site-controller/src/default_runtime/dynamic_routes.rs +++ b/runtime/site-controller/src/default_runtime/dynamic_routes.rs @@ -1325,6 +1325,7 @@ mod direct_input_tests { kubernetes_namespace: Some("amber-test-kind-local".to_string()), context: None, observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, launch_env: BTreeMap::new(), } } @@ -1370,6 +1371,7 @@ mod direct_input_tests { kubernetes_namespace: None, context: None, observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, launch_env: BTreeMap::new(), }; @@ -1421,6 +1423,7 @@ mod direct_input_tests { kubernetes_namespace: Some("amber-test-kind-local".to_string()), context: None, observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, launch_env: BTreeMap::new(), }; @@ -1862,6 +1865,7 @@ mod direct_input_tests { kubernetes_namespace: None, context: None, observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, launch_env: BTreeMap::new(), }; let child = SiteControllerRuntimeChildRecord { diff --git a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs index 064e6f19..24c88dff 100644 --- a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs +++ b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs @@ -913,6 +913,7 @@ mod tests { kubernetes_namespace: Some("test-ns".to_string()), context: Some("test-context".to_string()), observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, launch_env: BTreeMap::from([( "AMBER_DEV_IMAGE_TAGS".to_string(), "router=dev-tag,helper=dev-tag".to_string(), diff --git a/runtime/site-controller/src/default_runtime/site_artifacts.rs b/runtime/site-controller/src/default_runtime/site_artifacts.rs index 34fc60d6..0f5f203e 100644 --- a/runtime/site-controller/src/default_runtime/site_artifacts.rs +++ b/runtime/site-controller/src/default_runtime/site_artifacts.rs @@ -1110,6 +1110,7 @@ mod tests { kubernetes_namespace: None, context: None, observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, launch_env: BTreeMap::new(), }) .expect("metadata collection should succeed without a runtime state file"); diff --git a/runtime/site-controller/src/default_runtime/site_runtime_support.rs b/runtime/site-controller/src/default_runtime/site_runtime_support.rs index bb2118b6..643ade47 100644 --- a/runtime/site-controller/src/default_runtime/site_runtime_support.rs +++ b/runtime/site-controller/src/default_runtime/site_runtime_support.rs @@ -228,7 +228,21 @@ pub fn observability_endpoint_for_site(kind: SiteKind, endpoint: &str) -> Result Ok(url.to_string()) } +#[derive(Clone, Copy)] +enum PortBindScope { + Loopback, + Host, +} + pub fn reserve_loopback_port() -> Result { + reserve_port(PortBindScope::Loopback) +} + +pub fn reserve_host_port() -> Result { + reserve_port(PortBindScope::Host) +} + +fn reserve_port(scope: PortBindScope) -> Result { const LOOPBACK_PORT_RANGE_START: u16 = 30000; const LOOPBACK_PORT_RANGE_END: u16 = 60000; static RESERVED_LOOPBACK_PORTS: OnceLock>> = OnceLock::new(); @@ -249,23 +263,27 @@ pub fn reserve_loopback_port() -> Result { if reserved.contains(&port) { continue; } - match TcpListener::bind(("127.0.0.1", port)) { - Ok(listener) => { - drop(listener); - reserved.insert(port); - return Ok(port); - } - Err(_) => continue, + if port_available(port, scope) { + reserved.insert(port); + return Ok(port); } } Err(miette::miette!( - "failed to allocate a unique loopback port in {}-{}", + "failed to allocate a unique local port in {}-{}", LOOPBACK_PORT_RANGE_START, LOOPBACK_PORT_RANGE_END - 1 )) } +fn port_available(port: u16, scope: PortBindScope) -> bool { + let addr = match scope { + PortBindScope::Loopback => "127.0.0.1", + PortBindScope::Host => "0.0.0.0", + }; + TcpListener::bind((addr, port)).is_ok() +} + pub(super) fn site_supervisor_plan_path(site_state_root: &Path) -> PathBuf { site_state_root.join("site-supervisor-plan.json") } @@ -1183,7 +1201,11 @@ pub(super) fn resolve_desktop_container_host_ip() -> Option { #[cfg(test)] mod tests { - use std::{collections::BTreeMap, fs, net::SocketAddr}; + use std::{ + collections::BTreeMap, + fs, + net::{SocketAddr, TcpListener}, + }; use amber_compiler::{ mesh::{PROXY_METADATA_FILENAME, PROXY_METADATA_VERSION}, @@ -1196,13 +1218,33 @@ mod tests { use tempfile::tempdir; use super::{ - LaunchedSite, SITE_PLAN_SCHEMA, SITE_PLAN_VERSION, SiteReceipt, SiteSupervisorPlan, - external_slot_url, resolve_link_external_url_for_output, router_mesh_addr_for_consumer, + LaunchedSite, PortBindScope, SITE_PLAN_SCHEMA, SITE_PLAN_VERSION, SiteReceipt, + SiteSupervisorPlan, external_slot_url, port_available, + resolve_link_external_url_for_output, router_mesh_addr_for_consumer, should_prepare_kubernetes_namespace, }; #[cfg(unix)] use super::{parse_process_status_code, process_tree_postorder_from_ps}; + #[test] + fn host_port_availability_rejects_ports_bound_on_non_primary_loopback() { + let Ok(listener) = TcpListener::bind(("127.0.0.2", 0)) else { + return; + }; + let port = listener + .local_addr() + .expect("listener local addr should be available") + .port(); + if !port_available(port, PortBindScope::Loopback) { + return; + } + + assert!( + !port_available(port, PortBindScope::Host), + "host-port reservations must reject ports that would make a 0.0.0.0 bind fail" + ); + } + fn kubernetes_supervisor_plan(context: Option<&str>) -> SiteSupervisorPlan { SiteSupervisorPlan { schema: SITE_PLAN_SCHEMA.to_string(), diff --git a/runtime/site-controller/src/lib.rs b/runtime/site-controller/src/lib.rs index ee9c188f..5c001d60 100644 --- a/runtime/site-controller/src/lib.rs +++ b/runtime/site-controller/src/lib.rs @@ -82,7 +82,7 @@ pub use self::{ cleanup_dynamic_site_children, host_service_bind_addr_for_consumer, inject_compose_site_controller, inject_kubernetes_site_controller, inject_site_controller_peer_router_routes, observability_endpoint_for_site, - prepare_kubernetes_artifact_namespace, reserve_loopback_port, + prepare_kubernetes_artifact_namespace, reserve_host_port, reserve_loopback_port, router_mesh_addr_for_consumer, set_compose_router_published_mesh_port, set_site_artifact_mesh_identity_seed, site_controller_peer_router_url, vm_endpoint_forward_ready_timeout, vm_uses_tcg_accel, walk_files, diff --git a/runtime/site-controller/src/runtime_api.rs b/runtime/site-controller/src/runtime_api.rs index 0eae1d69..a91c74df 100644 --- a/runtime/site-controller/src/runtime_api.rs +++ b/runtime/site-controller/src/runtime_api.rs @@ -185,6 +185,8 @@ pub struct SiteControllerRuntimePlan { pub context: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub observability_endpoint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vm_endpoint_forward_ready_timeout_secs: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub launch_env: BTreeMap, } @@ -253,6 +255,7 @@ pub fn site_controller_runtime_plan_from_controller_plan( kubernetes_namespace: plan.kubernetes_namespace.clone(), context: plan.context.clone(), observability_endpoint: plan.observability_endpoint.clone(), + vm_endpoint_forward_ready_timeout_secs: plan.vm_endpoint_forward_ready_timeout_secs, launch_env: plan.launch_env.clone(), } } diff --git a/runtime/site-controller/src/state.rs b/runtime/site-controller/src/state.rs index 89aa3797..f6cdbea1 100644 --- a/runtime/site-controller/src/state.rs +++ b/runtime/site-controller/src/state.rs @@ -289,6 +289,8 @@ pub struct SiteControllerPlan { pub context: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub observability_endpoint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vm_endpoint_forward_ready_timeout_secs: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub launch_env: BTreeMap, } @@ -1283,6 +1285,9 @@ pub fn write_site_controller_plan( kubernetes_namespace: kubernetes_namespace.map(str::to_string), context: context.map(str::to_string), observability_endpoint: observability_endpoint.map(str::to_string), + vm_endpoint_forward_ready_timeout_secs: Some( + crate::default_runtime::vm_endpoint_forward_ready_timeout().as_secs(), + ), launch_env: launch_env.clone(), }; write_json(path, &plan)?; diff --git a/runtime/site-controller/src/tests.rs b/runtime/site-controller/src/tests.rs index 36940a88..4e25faf0 100644 --- a/runtime/site-controller/src/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -2752,6 +2752,7 @@ fn test_control_state_app( kubernetes_namespace: None, context: None, observability_endpoint: None, + vm_endpoint_forward_ready_timeout_secs: None, launch_env: BTreeMap::new(), }), authority_locks: Arc::new(Mutex::new(BTreeMap::new())), From 45087a8ed87646820f0c35fbf8f55943beaab0ae Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Sun, 3 May 2026 09:16:57 +0800 Subject: [PATCH 15/25] Avoid ephemeral ports for runtime listeners --- .../default_runtime/site_runtime_support.rs | 102 +++++++++++++++++- 1 file changed, 99 insertions(+), 3 deletions(-) diff --git a/runtime/site-controller/src/default_runtime/site_runtime_support.rs b/runtime/site-controller/src/default_runtime/site_runtime_support.rs index 643ade47..b4f729d9 100644 --- a/runtime/site-controller/src/default_runtime/site_runtime_support.rs +++ b/runtime/site-controller/src/default_runtime/site_runtime_support.rs @@ -234,6 +234,18 @@ enum PortBindScope { Host, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct PortRange { + start: u16, + end: u16, +} + +impl PortRange { + fn contains(self, port: u16) -> bool { + self.start <= port && port <= self.end + } +} + pub fn reserve_loopback_port() -> Result { reserve_port(PortBindScope::Loopback) } @@ -244,13 +256,14 @@ pub fn reserve_host_port() -> Result { fn reserve_port(scope: PortBindScope) -> Result { const LOOPBACK_PORT_RANGE_START: u16 = 30000; - const LOOPBACK_PORT_RANGE_END: u16 = 60000; + const LOOPBACK_PORT_RANGE_END: u16 = 65000; static RESERVED_LOOPBACK_PORTS: OnceLock>> = OnceLock::new(); let reserved = RESERVED_LOOPBACK_PORTS.get_or_init(|| std::sync::Mutex::new(BTreeSet::new())); let mut reserved = reserved .lock() .expect("loopback port allocator should not be poisoned"); + let ephemeral_range = host_ephemeral_port_range(); let span = u32::from(LOOPBACK_PORT_RANGE_END - LOOPBACK_PORT_RANGE_START); let mut next = LOOPBACK_PORT_RANGE_START + (std::process::id() % span) as u16 + reserved.len() as u16; @@ -263,6 +276,11 @@ fn reserve_port(scope: PortBindScope) -> Result { if reserved.contains(&port) { continue; } + // These ports become listeners after allocation; avoid ports the kernel may choose first + // for local outbound clients while the runtime is still starting. + if !port_allowed_for_runtime_listener(port, ephemeral_range) { + continue; + } if port_available(port, scope) { reserved.insert(port); return Ok(port); @@ -276,6 +294,54 @@ fn reserve_port(scope: PortBindScope) -> Result { )) } +fn port_allowed_for_runtime_listener(port: u16, ephemeral_range: Option) -> bool { + !ephemeral_range.is_some_and(|range| range.contains(port)) +} + +#[cfg(target_os = "linux")] +fn host_ephemeral_port_range() -> Option { + parse_linux_ephemeral_port_range( + &fs::read_to_string("/proc/sys/net/ipv4/ip_local_port_range").ok()?, + ) +} + +#[cfg(target_os = "linux")] +fn parse_linux_ephemeral_port_range(raw: &str) -> Option { + let mut parts = raw.split_whitespace(); + let start = parts.next()?.parse::().ok()?; + let end = parts.next()?.parse::().ok()?; + (start <= end).then_some(PortRange { start, end }) +} + +#[cfg(target_os = "macos")] +fn host_ephemeral_port_range() -> Option { + let start = sysctl_port("net.inet.ip.portrange.first")?; + let end = sysctl_port("net.inet.ip.portrange.last")?; + (start <= end).then_some(PortRange { start, end }) +} + +#[cfg(target_os = "macos")] +fn sysctl_port(name: &str) -> Option { + let name = std::ffi::CString::new(name).ok()?; + let mut value: libc::c_int = 0; + let mut len = std::mem::size_of_val(&value); + let rc = unsafe { + libc::sysctlbyname( + name.as_ptr(), + (&mut value as *mut libc::c_int).cast(), + &mut len, + std::ptr::null_mut(), + 0, + ) + }; + (rc == 0 && (0..=u16::MAX as libc::c_int).contains(&value)).then_some(value as u16) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn host_ephemeral_port_range() -> Option { + None +} + fn port_available(port: u16, scope: PortBindScope) -> bool { let addr = match scope { PortBindScope::Loopback => "127.0.0.1", @@ -1217,15 +1283,45 @@ mod tests { use base64::Engine as _; use tempfile::tempdir; + #[cfg(target_os = "linux")] + use super::parse_linux_ephemeral_port_range; use super::{ - LaunchedSite, PortBindScope, SITE_PLAN_SCHEMA, SITE_PLAN_VERSION, SiteReceipt, - SiteSupervisorPlan, external_slot_url, port_available, + LaunchedSite, PortBindScope, PortRange, SITE_PLAN_SCHEMA, SITE_PLAN_VERSION, SiteReceipt, + SiteSupervisorPlan, external_slot_url, port_allowed_for_runtime_listener, port_available, resolve_link_external_url_for_output, router_mesh_addr_for_consumer, should_prepare_kubernetes_namespace, }; #[cfg(unix)] use super::{parse_process_status_code, process_tree_postorder_from_ps}; + #[test] + fn runtime_port_allocator_rejects_ephemeral_client_ports() { + let linux_default = Some(PortRange { + start: 32768, + end: 60999, + }); + + assert!( + !port_allowed_for_runtime_listener(46274, linux_default), + "runtime listeners must not use ports the kernel can choose for outbound clients" + ); + assert!(port_allowed_for_runtime_listener(30000, linux_default)); + assert!(port_allowed_for_runtime_listener(61000, linux_default)); + } + + #[cfg(target_os = "linux")] + #[test] + fn parses_linux_ephemeral_port_range() { + assert_eq!( + parse_linux_ephemeral_port_range("32768\t60999\n"), + Some(PortRange { + start: 32768, + end: 60999 + }) + ); + assert_eq!(parse_linux_ephemeral_port_range("60999 32768"), None); + } + #[test] fn host_port_availability_rejects_ports_bound_on_non_primary_loopback() { let Ok(listener) = TcpListener::bind(("127.0.0.2", 0)) else { From 83348aa0529a0e694baff80d6393f96798601888 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Mon, 4 May 2026 12:26:28 +0800 Subject: [PATCH 16/25] address reivew comment --- compiler/src/run_plan.rs | 274 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 255 insertions(+), 19 deletions(-) diff --git a/compiler/src/run_plan.rs b/compiler/src/run_plan.rs index 9145a8b4..0d481442 100644 --- a/compiler/src/run_plan.rs +++ b/compiler/src/run_plan.rs @@ -2714,7 +2714,7 @@ fn framework_component_managed_site_id( fn framework_component_controller_provide_name(grant: &FrameworkComponentGrantKey) -> String { format!( "{FRAMEWORK_COMPONENT_CONTROLLER_PROVIDE_PREFIX}__site_{}__authority_{}", - sanitize_framework_component_identifier(grant.managed_site.as_str()), + framework_component_site_token(grant.managed_site.as_str()), grant.authority_realm.0, ) } @@ -2722,7 +2722,7 @@ fn framework_component_controller_provide_name(grant: &FrameworkComponentGrantKe pub fn framework_component_controller_remote_slot_name(site_id: &str) -> String { format!( "{FRAMEWORK_COMPONENT_CONTROLLER_REMOTE_SLOT_PREFIX}__{}", - sanitize_framework_component_identifier(site_id), + framework_component_site_token(site_id), ) } @@ -2732,7 +2732,7 @@ fn unique_framework_component_controller_moniker( ) -> String { let base = format!( "{FRAMEWORK_COMPONENT_CONTROLLER_MONIKER_PREFIX}/{}", - sanitize_framework_component_identifier(execution_site), + framework_component_site_token(execution_site), ); let used = scenario .components_iter() @@ -2751,20 +2751,18 @@ fn unique_framework_component_controller_moniker( } } -fn sanitize_framework_component_identifier(raw: &str) -> String { - let mut out = String::with_capacity(raw.len()); - for ch in raw.chars() { - if ch.is_ascii_alphanumeric() { - out.push(ch.to_ascii_lowercase()); - } else { - out.push('_'); - } - } - if out.is_empty() { - "internal".to_string() - } else { - out +fn framework_component_site_token(site_id: &str) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + + // These tokens are embedded in synthetic slot/provide map keys, so lossy sanitization is not + // enough: distinct authored site ids must remain distinct after lowering. + let mut out = String::with_capacity("site_".len() + site_id.len() * 2); + out.push_str("site_"); + for byte in site_id.bytes() { + out.push(HEX[(byte >> 4) as usize] as char); + out.push(HEX[(byte & 0x0f) as usize] as char); } + out } fn next_framework_component_controller_port( @@ -3094,6 +3092,21 @@ mod tests { }"# } + fn framework_admin_image_manifest() -> &'static str { + r#"{ + manifest_version: "0.3.0", + slots: { ctl: { kind: "component" } }, + program: { + image: "busybox:1.36.1", + entrypoint: ["sh", "-c", "sleep 30"], + env: { CTL_URL: "${slots.ctl.url}" }, + network: { endpoints: [{ name: "http", port: 8080, protocol: "http" }] } + }, + provides: { api: { kind: "http", endpoint: "http" } }, + exports: { api: "api" } +}"# + } + fn framework_admin_vm_manifest() -> &'static str { r##"{ manifest_version: "0.3.0", @@ -3187,6 +3200,13 @@ mod tests { ) } + fn framework_controller_moniker_for_site(site_id: &str) -> String { + format!( + "{FRAMEWORK_COMPONENT_CONTROLLER_MONIKER_PREFIX}/{}", + framework_component_site_token(site_id) + ) + } + async fn compile(root: &Path) -> CompiledScenario { let compiler = Compiler::new(Resolver::new(), DigestStore::default()) .with_registry(ResolverRegistry::default()); @@ -3430,7 +3450,7 @@ mod tests { assert!(plan.sites.contains_key("guest_vm")); assert_eq!( plan.sites["host_direct"].assigned_components, - vec!["/__amber_internal_framework_component_controller/host_direct".to_string()] + vec![framework_controller_moniker_for_site("host_direct")] ); assert_eq!( plan.sites["guest_vm"].assigned_components, @@ -3656,6 +3676,118 @@ mod tests { assert!(plan.links.is_empty()); } + #[tokio::test] + async fn framework_component_lowering_keeps_colliding_managed_site_grants_distinct() { + let dir = tmp_dir("run-plan-framework-component-colliding-grants-"); + let admin = dir.path().join("admin.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_vm_manifest()); + write( + &root, + r###"{ + manifest_version: "0.3.0", + components: { + dash: "./admin.json5", + underscore: "./admin.json5" + }, + bindings: [ + { to: "#dash.ctl", from: "framework.component" }, + { to: "#underscore.ctl", from: "framework.component" } + ], + exports: { + dash_api: "#dash.api", + underscore_api: "#underscore.api" + } +}"###, + ); + let placement = PlacementFile { + schema: PLACEMENT_SCHEMA.to_string(), + version: PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "direct_local".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "site-a".to_string(), + SiteDefinition { + kind: SiteKind::Vm, + context: None, + controller_site: Some("direct_local".to_string()), + }, + ), + ( + "site_a".to_string(), + SiteDefinition { + kind: SiteKind::Vm, + context: None, + controller_site: Some("direct_local".to_string()), + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_local".to_string()), + image: None, + vm: Some("site-a".to_string()), + }, + components: BTreeMap::from([ + ("/dash".to_string(), "site-a".to_string()), + ("/underscore".to_string(), "site_a".to_string()), + ]), + dynamic_capabilities: None, + framework_children: None, + }; + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, Some(&placement)).expect("run plan should build"); + let lowered = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered base scenario should deserialize"); + let scenario = lowered.scenario(); + let (controller_id, controller, metadata) = + framework_controller_component_for_site(scenario, "direct_local"); + let managed_sites = metadata + .grants + .values() + .map(|grant| grant.managed_site.clone()) + .collect::>(); + + assert_eq!( + managed_sites, + BTreeSet::from(["site-a".to_string(), "site_a".to_string()]), + "colliding sanitized site ids must not overwrite each other's controller grants", + ); + assert_eq!( + controller.provides.len(), + 3, + "controller should expose one internal provide plus one provide per managed site", + ); + + for (moniker, expected_site) in [("/dash", "site-a"), ("/underscore", "site_a")] { + let binding = scenario + .bindings + .iter() + .find(|binding| { + graph::component_path(scenario, binding.to.component) == moniker + && binding.to.name == "ctl" + }) + .expect("framework binding should remain present"); + let BindingFrom::Component(provide) = &binding.from else { + panic!("framework binding should lower to a synthetic controller provide"); + }; + assert_eq!(provide.component, controller_id); + let grant = metadata + .grants + .get(&provide.name) + .expect("lowered controller provide should carry grant metadata"); + assert_eq!(grant.managed_site, expected_site); + } + } + #[tokio::test] async fn framework_component_lowering_reuses_existing_synthetic_controller_on_replay() { let dir = tmp_dir("run-plan-framework-component-replay-"); @@ -3900,6 +4032,110 @@ mod tests { ); } + #[tokio::test] + async fn framework_component_lowering_keeps_colliding_remote_controller_slots_distinct() { + let dir = tmp_dir("run-plan-framework-component-colliding-remote-slots-"); + let admin = dir.path().join("admin.json5"); + let root = dir.path().join("root.json5"); + + write(&admin, framework_admin_image_manifest()); + write( + &root, + r##"{ + manifest_version: "0.3.0", + slots: { realm: { kind: "component", optional: true } }, + components: { admin: "./admin.json5" }, + child_templates: { + worker: { + possible_backends: ["direct"] + } + }, + bindings: [ + { to: "#admin.ctl", from: "framework.component" } + ], + exports: { admin_api: "#admin.api" } +}"##, + ); + let placement = PlacementFile { + schema: PLACEMENT_SCHEMA.to_string(), + version: PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "compose_local".to_string(), + SiteDefinition { + kind: SiteKind::Compose, + context: None, + controller_site: None, + }, + ), + ( + "site-a".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "site_a".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("site-a".to_string()), + image: Some("compose_local".to_string()), + vm: None, + }, + components: BTreeMap::new(), + dynamic_capabilities: None, + framework_children: None, + }; + + let compiled = compile(&root).await; + let plan = build_run_plan(&compiled, Some(&placement)).expect("run plan should build"); + let lowered = CompiledScenario::from_ir(plan.base_scenario.clone()) + .expect("lowered scenario should deserialize"); + let scenario = lowered.scenario(); + let (compose_controller_id, _, _) = + framework_controller_component_for_site(scenario, "compose_local"); + let site_dash_slot = framework_component_controller_remote_slot_name("site-a"); + let site_underscore_slot = framework_component_controller_remote_slot_name("site_a"); + + assert_ne!( + site_dash_slot, site_underscore_slot, + "remote controller slots must preserve distinct authored site ids", + ); + let remote_bindings = scenario + .bindings + .iter() + .filter(|binding| { + binding.to.component == compose_controller_id + && binding + .to + .name + .starts_with(FRAMEWORK_COMPONENT_CONTROLLER_REMOTE_SLOT_PREFIX) + }) + .collect::>(); + let remote_slots = remote_bindings + .iter() + .map(|binding| binding.to.name.clone()) + .collect::>(); + assert_eq!( + remote_bindings.len(), + 2, + "compose controller should bind to both colliding direct-site controllers", + ); + assert_eq!( + remote_slots, + BTreeSet::from([site_dash_slot, site_underscore_slot]), + "remote controller routes must not collapse onto one synthetic slot", + ); + } + #[tokio::test] async fn framework_component_lowering_routes_vm_template_creators_to_controller_site() { let dir = tmp_dir("run-plan-framework-component-vm-template-controller-"); @@ -4273,12 +4509,12 @@ mod tests { plan.sites["compose_local"].assigned_components, vec![ "/".to_string(), - "/__amber_internal_framework_component_controller/compose_local".to_string(), + framework_controller_moniker_for_site("compose_local"), ] ); assert_eq!( plan.sites["direct_local"].assigned_components, - vec!["/__amber_internal_framework_component_controller/direct_local".to_string()] + vec![framework_controller_moniker_for_site("direct_local")] ); assert!( plan.sites["direct_local"] From 88c1e8ada40aa269f41d79fa71702db7c0e19022 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Mon, 4 May 2026 15:49:58 +0800 Subject: [PATCH 17/25] Clean up deps --- Cargo.lock | 53 ------------------------ cli/Cargo.toml | 2 +- compiler/config/Cargo.toml | 1 - compiler/config/src/schema.rs | 22 ++++++++-- runtime/docker-gateway/Cargo.toml | 1 - runtime/docker-gateway/src/lib.rs | 33 ++++++++------- runtime/docker-gateway/src/tests/mod.rs | 3 +- runtime/docker-gateway/src/tests/unit.rs | 9 ++-- runtime/site-controller/Cargo.toml | 1 - 9 files changed, 41 insertions(+), 84 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab0dd3f1..2f80f6cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -177,7 +177,6 @@ dependencies = [ "amber-template", "jsonptr", "jsonschema", - "serde_jcs", "serde_json", ] @@ -195,7 +194,6 @@ dependencies = [ "moka", "serde", "serde_json", - "serde_urlencoded", "serde_with", "tempfile", "thiserror 2.0.18", @@ -438,7 +436,6 @@ dependencies = [ "serde_json", "serde_yaml", "sha2", - "tar", "tempfile", "tokio", "url", @@ -1466,7 +1463,6 @@ checksum = "05669f8e7e2d7badc545c513710f0eba09c2fbef683eb859fd79c46c355048e0" dependencies = [ "bitflags 1.3.2", "byteorder", - "chrono", "log", ] @@ -1476,17 +1472,6 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" -[[package]] -name = "filetime" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" -dependencies = [ - "cfg-if", - "libc", - "libredox", -] - [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -3465,12 +3450,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "ryu-js" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6518fc26bced4d53678a22d6e423e9d8716377def84545fe328236e3af070e7f" - [[package]] name = "same-file" version = "1.0.6" @@ -3607,17 +3586,6 @@ dependencies = [ "syn", ] -[[package]] -name = "serde_jcs" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cacecf649bc1a7c5f0e299cc813977c6a78116abda2b93b1ee01735b71ead9a8" -dependencies = [ - "ryu-js", - "serde", - "serde_json", -] - [[package]] name = "serde_json" version = "1.0.149" @@ -4168,17 +4136,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" -[[package]] -name = "tar" -version = "0.4.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" -dependencies = [ - "filetime", - "libc", - "xattr", -] - [[package]] name = "tempfile" version = "3.26.0" @@ -5441,16 +5398,6 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" -[[package]] -name = "xattr" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" -dependencies = [ - "libc", - "rustix", -] - [[package]] name = "yoke" version = "0.8.1" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 4e633236..eac9cde3 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -23,7 +23,7 @@ axum = { version = "0.8.6", features = ["http1", "json", "tokio"] } base64 = { workspace = true } clap = { version = "4.5.46", features = ["derive"] } crossterm = "0.29.0" -fatfs = "0.3.6" +fatfs = { version = "=0.3.6", default-features = false, features = ["std", "alloc"] } jsonschema = { workspace = true, default-features = false } libc = "0.2.177" miette = { workspace = true, features = ["fancy"] } diff --git a/compiler/config/Cargo.toml b/compiler/config/Cargo.toml index 27cfbdc7..fec0b76c 100644 --- a/compiler/config/Cargo.toml +++ b/compiler/config/Cargo.toml @@ -7,4 +7,3 @@ amber-template = { workspace = true } jsonptr = { workspace = true } jsonschema = { workspace = true } serde_json = { workspace = true } -serde_jcs = "0.1.0" diff --git a/compiler/config/src/schema.rs b/compiler/config/src/schema.rs index 579e0161..1adeb1f8 100644 --- a/compiler/config/src/schema.rs +++ b/compiler/config/src/schema.rs @@ -1470,10 +1470,24 @@ fn ensure_leaf_schema_supported(schema: &Map) -> Result<()> { } pub fn canonical_json(v: &Value) -> Value { - let canonical_bytes = - serde_jcs::to_vec(v).expect("serializing config schema to canonical JSON should succeed"); - serde_json::from_slice(&canonical_bytes) - .expect("canonical JSON bytes should deserialize back into JSON value") + match v { + Value::Array(values) => Value::Array(values.iter().map(canonical_json).collect()), + Value::Object(map) => { + let mut entries: Vec<_> = map.iter().collect(); + entries.sort_by(|(left, _), (right, _)| canonical_json_key_order(left, right)); + Value::Object( + entries + .into_iter() + .map(|(key, value)| (key.clone(), canonical_json(value))) + .collect(), + ) + } + _ => v.clone(), + } +} + +fn canonical_json_key_order(left: &str, right: &str) -> std::cmp::Ordering { + left.encode_utf16().cmp(right.encode_utf16()) } // Produce a minimized schema that only includes explicitly allowed leaf paths. diff --git a/runtime/docker-gateway/Cargo.toml b/runtime/docker-gateway/Cargo.toml index ca71e1f9..d2ad1a83 100644 --- a/runtime/docker-gateway/Cargo.toml +++ b/runtime/docker-gateway/Cargo.toml @@ -13,7 +13,6 @@ hyperlocal = { version = "0.9.1", features = ["client"] } moka = { version = "0.12.10", features = ["sync"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } -serde_urlencoded = "0.7.1" serde_with = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net", "io-util", "sync", "time", "signal"] } diff --git a/runtime/docker-gateway/src/lib.rs b/runtime/docker-gateway/src/lib.rs index 59ed24a1..672ecbb9 100644 --- a/runtime/docker-gateway/src/lib.rs +++ b/runtime/docker-gateway/src/lib.rs @@ -629,7 +629,7 @@ async fn handle(mut req: Request, conn: Arc) -> Response

>(q).ok()) + .map(parse_query_pairs) .map(|params| { params.iter().any(|(k, _)| k == "fromImage") && !params.iter().any(|(k, _)| k == "fromSrc") @@ -916,8 +916,7 @@ fn build_label_filter_query(required_labels: &[String], include_all: bool) -> St } query_pairs.push(("filters".to_string(), encoded_filters)); - serde_urlencoded::to_string(query_pairs) - .expect("serializing static label query params should succeed") + encode_query_pairs(&query_pairs) } async fn shutdown_signal() -> ShutdownReason { @@ -1520,15 +1519,24 @@ fn ensure_host(req: &mut Request) { } } +fn parse_query_pairs(query: &str) -> Vec<(String, String)> { + url::form_urlencoded::parse(query.as_bytes()) + .into_owned() + .collect() +} + +fn encode_query_pairs(pairs: &[(String, String)]) -> String { + let mut serializer = url::form_urlencoded::Serializer::new(String::new()); + for (key, value) in pairs { + serializer.append_pair(key, value); + } + serializer.finish() +} + fn add_label_filters_to_uri(uri: &Uri, required: &[String]) -> GatewayResult { let path = uri.path().to_string(); let mut pairs: Vec<(String, String)> = match uri.query() { - Some(query) => serde_urlencoded::from_str(query).map_err(|err| { - boxed_response(docker_error( - StatusCode::BAD_REQUEST, - format!("invalid query parameters: {err}"), - )) - })?, + Some(query) => parse_query_pairs(query), None => Vec::new(), }; @@ -1570,12 +1578,7 @@ fn add_label_filters_to_uri(uri: &Uri, required: &[String]) -> GatewayResult serde_json::Value { .path_and_query .split_once('?') .expect("request should include query"); - let query_map: HashMap = - serde_urlencoded::from_str(query).expect("query should decode"); + let query_map: HashMap = parse_query_pairs(query).into_iter().collect(); let filters = query_map.get("filters").expect("filters query param"); serde_json::from_str(filters).expect("filters should be valid json") } diff --git a/runtime/docker-gateway/src/tests/unit.rs b/runtime/docker-gateway/src/tests/unit.rs index 0274ac05..a5075ff7 100644 --- a/runtime/docker-gateway/src/tests/unit.rs +++ b/runtime/docker-gateway/src/tests/unit.rs @@ -50,8 +50,7 @@ fn add_label_filters_to_uri_accepts_object_labels() { let required = vec!["c=d".to_string()]; let out = add_label_filters_to_uri(&uri, &required).expect("filters"); let query = out.query().expect("query"); - let parsed_query: HashMap = - serde_urlencoded::from_str(query).expect("query should decode"); + let parsed_query: HashMap = parse_query_pairs(query).into_iter().collect(); let filters = parsed_query.get("filters").expect("filters"); let parsed_filters: serde_json::Value = serde_json::from_str(filters).expect("json"); let labels = parsed_filters @@ -77,8 +76,7 @@ fn add_label_filters_to_uri_uses_object_labels_with_name_filter() { let required = vec!["amber.component=/green".to_string()]; let out = add_label_filters_to_uri(&uri, &required).expect("filters"); let query = out.query().expect("query"); - let parsed_query: HashMap = - serde_urlencoded::from_str(query).expect("query should decode"); + let parsed_query: HashMap = parse_query_pairs(query).into_iter().collect(); let filters = parsed_query.get("filters").expect("filters"); let parsed_filters: serde_json::Value = serde_json::from_str(filters).expect("json"); @@ -121,8 +119,7 @@ fn add_label_filters_to_uri_rewrites_compose_project_filter() { let out = add_label_filters_to_uri(&uri, &required).expect("filters"); let query = out.query().expect("query"); - let parsed_query: HashMap = - serde_urlencoded::from_str(query).expect("query should decode"); + let parsed_query: HashMap = parse_query_pairs(query).into_iter().collect(); let filters = parsed_query.get("filters").expect("filters"); let parsed_filters: serde_json::Value = serde_json::from_str(filters).expect("json"); let labels = parsed_filters diff --git a/runtime/site-controller/Cargo.toml b/runtime/site-controller/Cargo.toml index 4feaf417..5b92951b 100644 --- a/runtime/site-controller/Cargo.toml +++ b/runtime/site-controller/Cargo.toml @@ -24,7 +24,6 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } serde_yaml = { workspace = true } sha2 = "0.10.9" -tar = "0.4.44" tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "signal", "time"] } url = { workspace = true } From 3f958962f754b93cd0976e587b8ff0357351a208 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Mon, 4 May 2026 16:34:54 +0800 Subject: [PATCH 18/25] Remove NULL bytes --- runtime/mesh/src/lib.rs | Bin 21670 -> 21939 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/runtime/mesh/src/lib.rs b/runtime/mesh/src/lib.rs index 99dbfe72b2dd6cf3564a41ec39f9abdc2f9c3b9b..ce378e5f6e3b992a05eee7d7aee58d69c128be5c 100644 GIT binary patch delta 259 zcmZ3sl5z8D#tr#0lO2sECb!9GPu|PUHQC#kZ}JHlH5hv{t8BROQQN{VCMail0 zDfy|z@p<_r@yYplC5f4N@j00#sYQu7@p+{=@kx~>sl^(a3e{W+P>_>aq5#pUV5^Xs zmz-0Yk_u6#sG*dTnWR@#tTg$gk3xnbM3o-Y*kTPe15F*cF(A{FN{dq!Qj3!l3sO@Q zVqy#wic Date: Mon, 4 May 2026 16:59:38 +0800 Subject: [PATCH 19/25] Improve reliability of controller requests through retries --- runtime/site-controller/src/http.rs | 92 +++++++++++++++++++ runtime/site-controller/src/orchestration.rs | 86 ----------------- .../site-controller/src/site_controller.rs | 21 ++--- runtime/site-controller/src/tests.rs | 68 +++++++++++++- 4 files changed, 169 insertions(+), 98 deletions(-) diff --git a/runtime/site-controller/src/http.rs b/runtime/site-controller/src/http.rs index 38795791..e19a7093 100644 --- a/runtime/site-controller/src/http.rs +++ b/runtime/site-controller/src/http.rs @@ -46,6 +46,39 @@ pub(super) fn required_header( }) } +const REMOTE_CONTROLLER_REQUEST_RETRY_TIMEOUT: Duration = Duration::from_secs(30); +const REMOTE_CONTROLLER_REQUEST_RETRY_DELAY: Duration = Duration::from_millis(250); + +fn should_retry_remote_controller_status(status: StatusCode) -> bool { + matches!( + status, + StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE | StatusCode::GATEWAY_TIMEOUT + ) +} + +pub(super) async fn post_json_with_retry( + client: &ReqwestClient, + url: &str, + body: &TReq, +) -> std::result::Result { + let deadline = tokio::time::Instant::now() + REMOTE_CONTROLLER_REQUEST_RETRY_TIMEOUT; + loop { + match client.post(url).json(body).send().await { + Ok(response) + if should_retry_remote_controller_status(response.status()) + && tokio::time::Instant::now() < deadline => + { + tokio::time::sleep(REMOTE_CONTROLLER_REQUEST_RETRY_DELAY).await; + } + Ok(response) => return Ok(response), + Err(_) if tokio::time::Instant::now() < deadline => { + tokio::time::sleep(REMOTE_CONTROLLER_REQUEST_RETRY_DELAY).await; + } + Err(err) => return Err(err), + } + } +} + pub(super) async fn shutdown_signal() { let ctrl_c = async { signal::ctrl_c() @@ -158,3 +191,62 @@ pub(super) fn read_json Deserialize<'de>>(path: &Path, label: &str) serde_json::from_slice(&bytes) .map_err(|err| miette::miette!("invalid {label} {}: {err}", path.display())) } + +#[cfg(test)] +mod tests { + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + + use axum::{ + Json, Router, + extract::State, + http::StatusCode, + response::{IntoResponse as _, Response}, + routing::post, + }; + use serde_json::json; + use tokio::net::TcpListener; + + use super::*; + + #[tokio::test] + async fn post_json_with_retry_tolerates_transient_service_unavailable() { + async fn handler(State(attempts): State>) -> Response { + let attempt = attempts.fetch_add(1, Ordering::SeqCst); + if attempt < 2 { + return (StatusCode::SERVICE_UNAVAILABLE, Json(json!({"ok": false}))) + .into_response(); + } + (StatusCode::OK, Json(json!({"ok": true}))).into_response() + } + + let attempts = Arc::new(AtomicUsize::new(0)); + let app = Router::new() + .route("/retry", post(handler)) + .with_state(attempts.clone()); + let listener = TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("listener should bind"); + let addr = listener.local_addr().expect("listener addr"); + let serve = tokio::spawn(async move { + axum::serve(listener, app) + .await + .expect("test server should run"); + }); + + let response = post_json_with_retry( + &ReqwestClient::new(), + &format!("http://{addr}/retry"), + &json!({"hello": "world"}), + ) + .await + .expect("request should eventually succeed"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(attempts.load(Ordering::SeqCst), 3); + + serve.abort(); + let _ = serve.await; + } +} diff --git a/runtime/site-controller/src/orchestration.rs b/runtime/site-controller/src/orchestration.rs index 77e19772..85ecf85c 100644 --- a/runtime/site-controller/src/orchestration.rs +++ b/runtime/site-controller/src/orchestration.rs @@ -731,39 +731,6 @@ fn remote_controller_base_url( }) } -const REMOTE_CONTROLLER_REQUEST_RETRY_TIMEOUT: Duration = Duration::from_secs(30); -const REMOTE_CONTROLLER_REQUEST_RETRY_DELAY: Duration = Duration::from_millis(250); - -fn should_retry_remote_controller_status(status: StatusCode) -> bool { - matches!( - status, - StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE | StatusCode::GATEWAY_TIMEOUT - ) -} - -async fn post_json_with_retry( - client: &ReqwestClient, - url: &str, - body: &TReq, -) -> std::result::Result { - let deadline = tokio::time::Instant::now() + REMOTE_CONTROLLER_REQUEST_RETRY_TIMEOUT; - loop { - match client.post(url).json(body).send().await { - Ok(response) - if should_retry_remote_controller_status(response.status()) - && tokio::time::Instant::now() < deadline => - { - tokio::time::sleep(REMOTE_CONTROLLER_REQUEST_RETRY_DELAY).await; - } - Ok(response) => return Ok(response), - Err(_) if tokio::time::Instant::now() < deadline => { - tokio::time::sleep(REMOTE_CONTROLLER_REQUEST_RETRY_DELAY).await; - } - Err(err) => return Err(err), - } - } -} - async fn remote_controller_post_json Deserialize<'de>>( app: &ControlStateApp, site_id: &str, @@ -2281,56 +2248,3 @@ pub(super) async fn recover_control_state(app: &ControlStateApp) -> Result<()> { } Ok(()) } - -#[cfg(test)] -mod tests { - use std::sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, - }; - - use axum::{Json, Router, extract::State, response::Response, routing::post}; - use serde_json::json; - use tokio::net::TcpListener; - - use super::*; - - #[tokio::test] - async fn post_json_with_retry_tolerates_transient_service_unavailable() { - async fn handler(State(attempts): State>) -> Response { - let attempt = attempts.fetch_add(1, Ordering::SeqCst); - if attempt < 2 { - return (StatusCode::SERVICE_UNAVAILABLE, Json(json!({"ok": false}))) - .into_response(); - } - (StatusCode::OK, Json(json!({"ok": true}))).into_response() - } - - let attempts = Arc::new(AtomicUsize::new(0)); - let app = Router::new() - .route("/retry", post(handler)) - .with_state(attempts.clone()); - let listener = TcpListener::bind(("127.0.0.1", 0)) - .await - .expect("listener should bind"); - let addr = listener.local_addr().expect("listener addr"); - let serve = tokio::spawn(async move { - axum::serve(listener, app) - .await - .expect("test server should run"); - }); - - let response = post_json_with_retry( - &ReqwestClient::new(), - &format!("http://{addr}/retry"), - &json!({"hello": "world"}), - ) - .await - .expect("request should eventually succeed"); - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(attempts.load(Ordering::SeqCst), 3); - - serve.abort(); - let _ = serve.await; - } -} diff --git a/runtime/site-controller/src/site_controller.rs b/runtime/site-controller/src/site_controller.rs index 5455c417..b0ed683b 100644 --- a/runtime/site-controller/src/site_controller.rs +++ b/runtime/site-controller/src/site_controller.rs @@ -27,7 +27,10 @@ use super::{ ControlDynamicRevokeRequest, ControlDynamicShareRequest, InternalDynamicResolveOriginRequest, }, - http::{cleanup_dynamic_bridge_proxies, read_json, required_header, shutdown_signal}, + http::{ + cleanup_dynamic_bridge_proxies, post_json_with_retry, read_json, required_header, + shutdown_signal, + }, orchestration::{ ClearExportPeerOverlayRequest, ClearExternalSlotOverlayRequest, ProtocolApiError, PublishExportPeerOverlayRequest, PublishExternalSlotOverlayRequest, @@ -515,16 +518,12 @@ async fn remote_controller_post( path: &str, body: &TReq, ) -> std::result::Result { - let response = app - .control - .client - .post(format!( - "{}{}", - remote_controller_base_url(app, site_id)?.trim_end_matches('/'), - path - )) - .json(body) - .send() + let url = format!( + "{}{}", + remote_controller_base_url(app, site_id)?.trim_end_matches('/'), + path + ); + let response = post_json_with_retry(&app.control.client, &url, body) .await .map_err(|err| { ProtocolApiError::control_state_unavailable(format!( diff --git a/runtime/site-controller/src/tests.rs b/runtime/site-controller/src/tests.rs index 4e25faf0..5ae30451 100644 --- a/runtime/site-controller/src/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -2,7 +2,10 @@ use std::{ fs, future::Future, io::{Read, Write}, - sync::OnceLock, + sync::{ + Arc, OnceLock, + atomic::{AtomicUsize, Ordering}, + }, time::Duration as StdDuration, }; @@ -2943,6 +2946,69 @@ fn with_controller_endpoint( } } +#[tokio::test] +async fn remote_dynamic_capability_controller_post_retries_transient_unavailable() { + async fn handler( + axum::extract::State(attempts): axum::extract::State>, + ) -> axum::response::Response { + use axum::response::IntoResponse as _; + + let attempt = attempts.fetch_add(1, Ordering::SeqCst); + if attempt < 2 { + return ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ + "code": "ControlStateUnavailable", + "message": "site controller is still recovering" + })), + ) + .into_response(); + } + (StatusCode::OK, Json(json!({"held": []}))).into_response() + } + + let attempts = Arc::new(AtomicUsize::new(0)); + let (url, handle) = spawn_test_router( + Router::new() + .route( + "/v1/controller/dynamic-caps/held", + axum::routing::post(handler), + ) + .with_state(attempts.clone()), + ) + .await; + let (dir, state, state_path) = compile_empty_control_state().await; + let mut control = test_control_state_app(&dir, state, state_path); + install_remote_controller_peer_url_fixture( + &mut control, + &BTreeMap::from([("remote".to_string(), url)]), + ); + let app = SiteControllerApp { + control, + ready: ready_site_controller_flag(), + }; + + let response = super::site_controller::execute_site_controller_dynamic_caps_inspect( + &app, + super::control_state_api::DynamicCapsInspectRequest::HeldList( + dynamic_caps::ControlDynamicHeldListRequest { + holder_component_id: "components./caller".to_string(), + }, + ), + false, + ) + .await + .expect("request should succeed after transient controller unavailability"); + let super::control_state_api::DynamicCapsInspectResponse::HeldList(response) = response else { + panic!("held_list should return held entries"); + }; + assert!(response.held.is_empty()); + assert_eq!(attempts.load(Ordering::SeqCst), 3); + + handle.abort(); + let _ = handle.await; +} + async fn install_framework_site_controller_fixture( app: &ControlStateApp, ) -> Vec> { From 69aab7ffd91c376896803a4d9026b83b4b1777f5 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Tue, 5 May 2026 00:29:48 +0800 Subject: [PATCH 20/25] Fix test --- cli/src/tests.rs | 5 +- cli/tests/mixed_run/dry_run.rs | 11 +- cli/tests/mixed_run/lifecycle.rs | 38 +++-- compiler/src/run_plan.rs | 150 +++++++++++++----- compiler/src/targets/mesh/mesh_config.rs | 4 +- .../src/default_runtime/dynamic_routes.rs | 22 +-- runtime/site-controller/src/tests.rs | 9 +- 7 files changed, 170 insertions(+), 69 deletions(-) diff --git a/cli/src/tests.rs b/cli/src/tests.rs index 078e6e41..68a72867 100644 --- a/cli/src/tests.rs +++ b/cli/src/tests.rs @@ -225,7 +225,10 @@ fn proxy_telemetry_keeps_router_info_without_verbose_output() { fn direct_runtime_only_isolates_non_controller_components() { let controller = DirectComponentPlan { id: 1, - moniker: "/__amber_internal_framework_component_controller/direct_local".to_string(), + moniker: amber_compiler::run_plan::FrameworkComponentControllerMoniker::for_site( + "direct_local", + ) + .into_string(), log_name: "controller".to_string(), source_dir: None, depends_on: Vec::new(), diff --git a/cli/tests/mixed_run/dry_run.rs b/cli/tests/mixed_run/dry_run.rs index f29e68e9..63d6edbb 100644 --- a/cli/tests/mixed_run/dry_run.rs +++ b/cli/tests/mixed_run/dry_run.rs @@ -1,5 +1,7 @@ use std::collections::BTreeSet; +use amber_compiler::run_plan::FrameworkComponentControllerMoniker; + use super::*; #[test] @@ -178,16 +180,15 @@ fn mixed_run_dry_run_emits_launch_bundle_without_starting_sites() { let stitching = launch_bundle["stitching"] .as_array() .expect("launch bundle stitching should serialize as an array"); - let controller_prefix = "/__amber_internal_framework_component_controller/"; let controller_stitching = stitching .iter() .filter(|link| { link["provider_component"] .as_str() - .is_some_and(|component| component.starts_with(controller_prefix)) + .is_some_and(FrameworkComponentControllerMoniker::is_synthetic_component) || link["consumer_component"] .as_str() - .is_some_and(|component| component.starts_with(controller_prefix)) + .is_some_and(FrameworkComponentControllerMoniker::is_synthetic_component) }) .collect::>(); let user_stitching = stitching @@ -195,10 +196,10 @@ fn mixed_run_dry_run_emits_launch_bundle_without_starting_sites() { .filter(|link| { !link["provider_component"] .as_str() - .is_some_and(|component| component.starts_with(controller_prefix)) + .is_some_and(FrameworkComponentControllerMoniker::is_synthetic_component) && !link["consumer_component"] .as_str() - .is_some_and(|component| component.starts_with(controller_prefix)) + .is_some_and(FrameworkComponentControllerMoniker::is_synthetic_component) }) .collect::>(); assert_eq!(user_stitching.len(), 1); diff --git a/cli/tests/mixed_run/lifecycle.rs b/cli/tests/mixed_run/lifecycle.rs index 83aec543..f7d4c760 100644 --- a/cli/tests/mixed_run/lifecycle.rs +++ b/cli/tests/mixed_run/lifecycle.rs @@ -1,3 +1,7 @@ +use std::collections::BTreeSet; + +use amber_compiler::run_plan::FrameworkComponentControllerMoniker; + use super::*; #[test] @@ -34,10 +38,11 @@ fn mixed_run_five_site_startup_state_and_teardown() { let assignments = run_plan["assignments"] .as_object() .expect("run plan assignments should be an object"); - let controller_prefix = "/__amber_internal_framework_component_controller/"; let user_assignments = assignments .iter() - .filter(|(component, _)| !component.starts_with(controller_prefix)) + .filter(|(component, _)| { + !FrameworkComponentControllerMoniker::is_synthetic_component(component) + }) .map(|(component, site)| (component.clone(), site.clone())) .collect::>(); assert_eq!( @@ -50,19 +55,26 @@ fn mixed_run_five_site_startup_state_and_teardown() { ("/e".to_string(), json!("compose_e")), ]) ); - let controller_assignments = assignments + let controller_assignment_sites = assignments .iter() - .filter(|(component, _)| component.starts_with(controller_prefix)) - .map(|(component, site)| (component.clone(), site.clone())) - .collect::>(); + .filter(|(component, _)| { + FrameworkComponentControllerMoniker::is_synthetic_component(component) + }) + .map(|(_, site)| { + site.as_str() + .expect("assignment site should be a string") + .to_string() + }) + .collect::>(); assert_eq!( - controller_assignments, - serde_json::Map::from_iter([ - (format!("{controller_prefix}compose_b"), json!("compose_b")), - (format!("{controller_prefix}compose_e"), json!("compose_e")), - (format!("{controller_prefix}direct_a"), json!("direct_a")), - (format!("{controller_prefix}kind_c"), json!("kind_c")), - ]) + controller_assignment_sites, + BTreeSet::from([ + "compose_b".to_string(), + "compose_e".to_string(), + "direct_a".to_string(), + "kind_c".to_string(), + ]), + "live startup should not depend on the private spelling of synthetic controller monikers", ); assert_eq!( run.receipt["sites"] diff --git a/compiler/src/run_plan.rs b/compiler/src/run_plan.rs index 0d481442..66b20d4f 100644 --- a/compiler/src/run_plan.rs +++ b/compiler/src/run_plan.rs @@ -1,5 +1,6 @@ use std::{ collections::{BTreeMap, BTreeSet, VecDeque}, + fmt, path::PathBuf, }; @@ -2321,6 +2322,87 @@ const FRAMEWORK_COMPONENT_CONTROLLER_CONTAINER_PATH: &str = "/usr/local/bin/ambe const FRAMEWORK_COMPONENT_CONTROLLER_PLAN_PATH: &str = "/amber/site/state/site-controller-plan.json"; +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct FrameworkComponentSiteToken(String); + +impl FrameworkComponentSiteToken { + pub fn for_site(site_id: &str) -> Self { + const HEX: &[u8; 16] = b"0123456789abcdef"; + + // These tokens are embedded in synthetic slot/provide map keys, so lossy sanitization is + // not enough: distinct authored site ids must remain distinct after lowering. + let mut out = String::with_capacity("site_".len() + site_id.len() * 2); + out.push_str("site_"); + for byte in site_id.bytes() { + out.push(HEX[(byte >> 4) as usize] as char); + out.push(HEX[(byte & 0x0f) as usize] as char); + } + Self(out) + } + + pub fn as_str(&self) -> &str { + self.0.as_str() + } + + pub fn controller_grant_provide_name(&self, authority_realm: ComponentId) -> String { + format!( + "{FRAMEWORK_COMPONENT_CONTROLLER_PROVIDE_PREFIX}__site_{}__authority_{}", + self.as_str(), + authority_realm.0, + ) + } + + pub fn controller_remote_slot_name(&self) -> String { + format!( + "{FRAMEWORK_COMPONENT_CONTROLLER_REMOTE_SLOT_PREFIX}__{}", + self.as_str(), + ) + } +} + +impl fmt::Display for FrameworkComponentSiteToken { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct FrameworkComponentControllerMoniker(String); + +impl FrameworkComponentControllerMoniker { + pub fn for_site(site_id: &str) -> Self { + Self(format!( + "{FRAMEWORK_COMPONENT_CONTROLLER_MONIKER_PREFIX}/{}", + FrameworkComponentSiteToken::for_site(site_id), + )) + } + + fn from_synthetic_string(moniker: String) -> Self { + debug_assert!(Self::is_synthetic_component(moniker.as_str())); + Self(moniker) + } + + pub fn as_str(&self) -> &str { + self.0.as_str() + } + + pub fn into_string(self) -> String { + self.0 + } + + pub fn is_synthetic_component(moniker: &str) -> bool { + moniker + .strip_prefix(FRAMEWORK_COMPONENT_CONTROLLER_MONIKER_PREFIX) + .is_some_and(|suffix| suffix.starts_with('/') && suffix.len() > 1) + } +} + +impl fmt::Display for FrameworkComponentControllerMoniker { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + fn has_public_framework_component_bindings(scenario: &Scenario) -> bool { scenario.bindings.iter().any(|binding| { matches!( @@ -2555,6 +2637,7 @@ fn inject_site_controller_components( id: controller_id, parent: Some(rewritten.root), moniker: unique_framework_component_controller_moniker(&rewritten, &execution_site) + .into_string() .into(), digest: rewritten.component(rewritten.root).digest, config: None, @@ -2712,28 +2795,19 @@ fn framework_component_managed_site_id( } fn framework_component_controller_provide_name(grant: &FrameworkComponentGrantKey) -> String { - format!( - "{FRAMEWORK_COMPONENT_CONTROLLER_PROVIDE_PREFIX}__site_{}__authority_{}", - framework_component_site_token(grant.managed_site.as_str()), - grant.authority_realm.0, - ) + FrameworkComponentSiteToken::for_site(grant.managed_site.as_str()) + .controller_grant_provide_name(grant.authority_realm) } pub fn framework_component_controller_remote_slot_name(site_id: &str) -> String { - format!( - "{FRAMEWORK_COMPONENT_CONTROLLER_REMOTE_SLOT_PREFIX}__{}", - framework_component_site_token(site_id), - ) + FrameworkComponentSiteToken::for_site(site_id).controller_remote_slot_name() } fn unique_framework_component_controller_moniker( scenario: &Scenario, execution_site: &str, -) -> String { - let base = format!( - "{FRAMEWORK_COMPONENT_CONTROLLER_MONIKER_PREFIX}/{}", - framework_component_site_token(execution_site), - ); +) -> FrameworkComponentControllerMoniker { + let base = FrameworkComponentControllerMoniker::for_site(execution_site); let used = scenario .components_iter() .map(|(_, component)| component.moniker.as_str()) @@ -2745,26 +2819,12 @@ fn unique_framework_component_controller_moniker( loop { let candidate = format!("{base}-{suffix}"); if !used.contains(candidate.as_str()) { - return candidate; + return FrameworkComponentControllerMoniker::from_synthetic_string(candidate); } suffix += 1; } } -fn framework_component_site_token(site_id: &str) -> String { - const HEX: &[u8; 16] = b"0123456789abcdef"; - - // These tokens are embedded in synthetic slot/provide map keys, so lossy sanitization is not - // enough: distinct authored site ids must remain distinct after lowering. - let mut out = String::with_capacity("site_".len() + site_id.len() * 2); - out.push_str("site_"); - for byte in site_id.bytes() { - out.push(HEX[(byte >> 4) as usize] as char); - out.push(HEX[(byte & 0x0f) as usize] as char); - } - out -} - fn next_framework_component_controller_port( used_ports: &mut BTreeSet, ) -> Result { @@ -3201,10 +3261,7 @@ mod tests { } fn framework_controller_moniker_for_site(site_id: &str) -> String { - format!( - "{FRAMEWORK_COMPONENT_CONTROLLER_MONIKER_PREFIX}/{}", - framework_component_site_token(site_id) - ) + FrameworkComponentControllerMoniker::for_site(site_id).into_string() } async fn compile(root: &Path) -> CompiledScenario { @@ -4729,16 +4786,37 @@ mod tests { "direct_local".to_string() ] ); - let controller_prefix = "/__amber_internal_framework_component_controller/"; let user_link_count = plan .links .iter() .filter(|link| { - !link.provider_component.starts_with(controller_prefix) - && !link.consumer_component.starts_with(controller_prefix) + !FrameworkComponentControllerMoniker::is_synthetic_component( + link.provider_component.as_str(), + ) && !FrameworkComponentControllerMoniker::is_synthetic_component( + link.consumer_component.as_str(), + ) }) .count(); assert_eq!(user_link_count, 6); + let controller_assignment_sites = plan + .assignments + .iter() + .filter_map(|(component, site)| { + FrameworkComponentControllerMoniker::is_synthetic_component(component) + .then_some(site.clone()) + }) + .collect::>(); + assert_eq!( + controller_assignment_sites, + BTreeSet::from([ + "compose_b".to_string(), + "compose_e".to_string(), + "direct_local".to_string(), + "kind_c".to_string(), + ]), + "the five-site run plan should assign synthetic controllers for every non-VM active \ + site before any live runtime work starts", + ); } #[test] diff --git a/compiler/src/targets/mesh/mesh_config.rs b/compiler/src/targets/mesh/mesh_config.rs index 429bf35b..1582bc8d 100644 --- a/compiler/src/targets/mesh/mesh_config.rs +++ b/compiler/src/targets/mesh/mesh_config.rs @@ -863,9 +863,11 @@ mod tests { "entrypoint": ["consumer"], }), ); + let controller_moniker = + crate::run_plan::FrameworkComponentControllerMoniker::for_site("site-a").into_string(); let mut controller = component( 1, - "/__amber_internal_framework_component_controller/site-a", + controller_moniker.as_str(), json!({ "image": "controller", "entrypoint": ["controller"], diff --git a/runtime/site-controller/src/default_runtime/dynamic_routes.rs b/runtime/site-controller/src/default_runtime/dynamic_routes.rs index 9cbf9246..dd8f05d3 100644 --- a/runtime/site-controller/src/default_runtime/dynamic_routes.rs +++ b/runtime/site-controller/src/default_runtime/dynamic_routes.rs @@ -1637,28 +1637,33 @@ mod direct_input_tests { #[test] fn dynamic_site_controller_overlay_uses_child_controller_route_grants() { + let controller_moniker = + amber_compiler::run_plan::FrameworkComponentControllerMoniker::for_site("site-a") + .into_string(); + let grant_provide = + amber_compiler::run_plan::FrameworkComponentSiteToken::for_site("site-a") + .controller_grant_provide_name(amber_scenario::ComponentId(0)); let internal_route_id = component_route_id( - "/__amber_internal_framework_component_controller/site-a", + controller_moniker.as_str(), amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, MeshProtocol::Http, ); let grant_route_id = component_route_id( - "/__amber_internal_framework_component_controller/site-a", - "__amber_internal_framework_component__site_site-a__authority_root", + controller_moniker.as_str(), + grant_provide.as_str(), MeshProtocol::Http, ); let live_components = BTreeMap::from([ ( - "/__amber_internal_framework_component_controller/site-a".to_string(), + controller_moniker.clone(), LiveComponentRuntimeMetadata { - moniker: "/__amber_internal_framework_component_controller/site-a".to_string(), + moniker: controller_moniker.clone(), router_reachable_mesh_addr: "127.0.0.1:24000".to_string(), component_reachable_mesh_addr: "127.0.0.1:24000".to_string(), control_endpoint: Some(ControlEndpoint::Unix("/tmp/controller.sock".into())), mesh_config: MeshConfigPublic { identity: MeshIdentityPublic { - id: "/__amber_internal_framework_component_controller/site-a" - .to_string(), + id: controller_moniker.clone(), public_key: [1; 32], mesh_scope: None, }, @@ -1725,8 +1730,7 @@ mod direct_input_tests { assigned_components: vec!["/job-dynamic".to_string()], controller_routes: vec![InboundRoute { route_id: grant_route_id.clone(), - capability: "__amber_internal_framework_component__site_site-a__authority_root" - .to_string(), + capability: grant_provide, capability_kind: Some("framework.component".to_string()), capability_profile: None, protocol: MeshProtocol::Http, diff --git a/runtime/site-controller/src/tests.rs b/runtime/site-controller/src/tests.rs index 5ae30451..c944e105 100644 --- a/runtime/site-controller/src/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -9,7 +9,7 @@ use std::{ time::Duration as StdDuration, }; -use amber_compiler::run_plan::build_run_plan; +use amber_compiler::run_plan::{FrameworkComponentControllerMoniker, build_run_plan}; use amber_mesh::{ InboundRoute, InboundTarget, MeshConfigPublic, MeshIdentityPublic, MeshPeer, MeshProtocol, OutboundRoute, TransportConfig, @@ -6258,7 +6258,8 @@ async fn create_child_publishes_sites_before_resolving_cross_site_link_overlays( scenario.components.push(Some(amber_scenario::Component { id: controller_id, parent: Some(root_id), - moniker: format!("/__amber_internal_framework_component_controller/{execution_site}") + moniker: FrameworkComponentControllerMoniker::for_site(execution_site) + .into_string() .into(), digest: scenario.component(root_id).digest, config: None, @@ -6796,8 +6797,8 @@ async fn local_controller_requests_select_the_controller_for_the_current_site() scenario.components.push(Some(amber_scenario::Component { id: compose_controller_id, parent: Some(root_id), - moniker: "/__amber_internal_framework_component_controller/compose_local" - .to_string() + moniker: FrameworkComponentControllerMoniker::for_site("compose_local") + .into_string() .into(), digest: scenario.component(root_id).digest, config: None, From 1255a84218ac9554d76c5e67ec0547318979b283 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Thu, 7 May 2026 23:26:40 +0800 Subject: [PATCH 21/25] Address review comments --- cli/src/command_support.rs | 329 ++++++++++++++++++ cli/src/direct_runtime/preview.rs | 7 +- cli/src/direct_runtime/runtime.rs | 7 +- cli/src/mixed_run.rs | 4 +- cli/src/mixed_run/launch_bundle.rs | 76 ++-- cli/src/mixed_run/supervisor.rs | 2 +- .../src/targets/mesh/docker_compose/mod.rs | 11 +- compiler/src/targets/mesh/kubernetes/mod.rs | 7 + .../mesh/kubernetes/tests/rendering.rs | 5 - compiler/src/targets/mesh/mesh_config.rs | 13 +- runtime/mesh/src/dynamic_caps.rs | 99 ++---- runtime/mesh/src/lib.rs | 2 - runtime/router/src/dynamic_caps.rs | 265 ++++---------- .../site-controller/src/default_runtime.rs | 4 +- .../src/default_runtime/child_runtime.rs | 4 +- .../src/default_runtime/compose_controller.rs | 23 +- .../default_runtime/kubernetes_controller.rs | 2 +- runtime/site-controller/src/dynamic_caps.rs | 59 +++- runtime/site-controller/src/lib.rs | 21 +- .../site-controller/src/site_controller.rs | 76 +++- runtime/site-controller/src/state.rs | 46 ++- runtime/site-controller/src/tests.rs | 327 ++++++++++++++--- 22 files changed, 978 insertions(+), 411 deletions(-) diff --git a/cli/src/command_support.rs b/cli/src/command_support.rs index 70d25e62..25861d55 100644 --- a/cli/src/command_support.rs +++ b/cli/src/command_support.rs @@ -571,9 +571,338 @@ pub(crate) fn write_unmanaged_export_output( unmanaged_export_output_dir_label(kind), &export.files, unmanaged_export_executable_rel_path(kind), + )?; + materialize_standalone_site_controller_export(root, run_plan, &export) +} + +fn materialize_standalone_site_controller_export( + root: &Path, + run_plan: &RunPlan, + export: &amber_compiler::run_plan::UnmanagedExport, +) -> Result<()> { + if !matches!(export.kind, SiteKind::Compose | SiteKind::Kubernetes) { + return Ok(()); + } + let site_plan = run_plan.sites.get(&export.site_id).ok_or_else(|| { + miette::miette!( + "unmanaged export references missing site `{}` in the run plan", + export.site_id + ) + })?; + let Some(controller_port) = crate::mixed_run::site_controller_component_port(site_plan)? else { + return Ok(()); + }; + + const COMPOSE_CONTROLLER_SITE_ROOT: &str = "/amber/site"; + const COMPOSE_CONTROLLER_STATE_ROOT: &str = "/amber/site/.amber/site-controller"; + + let artifact_root = root + .canonicalize() + .into_diagnostic() + .wrap_err_with(|| format!("failed to canonicalize export root `{}`", root.display()))?; + let base_run_id = format!( + "standalone-{}-{}", + unmanaged_export_label(export.kind), + export.site_id + ); + let kubernetes_namespace = if export.kind == SiteKind::Kubernetes { + Some( + amber_site_controller::prepare_kubernetes_artifact_namespace( + &base_run_id, + &export.site_id, + &artifact_root, + )?, + ) + } else { + None + }; + let run_id = kubernetes_namespace + .as_deref() + .unwrap_or(base_run_id.as_str()); + amber_site_controller::set_site_artifact_mesh_identity_seed(&artifact_root, run_id)?; + + let state_root = if export.kind == SiteKind::Kubernetes { + standalone_controller_temp_state_root(&export.site_id)? + } else { + artifact_root.join(".amber").join("site-controller") + }; + let site_state_root = state_root.join(&export.site_id); + crate::mixed_run::prepare_site_state_root(&site_state_root, export.kind)?; + let state_path = site_state_root.join("site-controller-state.json"); + let plan_path = crate::mixed_run::site_controller_plan_path(&site_state_root); + let (plan_state_path, plan_run_root, plan_state_root, plan_site_state_root, plan_artifact_root) = + if export.kind == SiteKind::Compose { + let plan_state_root = PathBuf::from(COMPOSE_CONTROLLER_STATE_ROOT); + let plan_site_state_root = plan_state_root.join(&export.site_id); + ( + plan_site_state_root.join("site-controller-state.json"), + PathBuf::from(COMPOSE_CONTROLLER_SITE_ROOT), + plan_state_root, + plan_site_state_root, + PathBuf::from(COMPOSE_CONTROLLER_SITE_ROOT), + ) + } else { + ( + state_path.clone(), + artifact_root.clone(), + state_root.clone(), + site_state_root.clone(), + artifact_root.clone(), + ) + }; + + let mut launch_env = BTreeMap::new(); + launch_env.insert( + amber_mesh::telemetry::SCENARIO_RUN_ID_ENV.to_string(), + run_id.to_string(), + ); + launch_env.insert( + amber_mesh::telemetry::SCENARIO_SCOPE_ENV.to_string(), + run_plan.mesh_scope.clone(), + ); + + let controller_state = amber_site_controller::build_site_controller_state( + run_id, + run_plan, + &export.site_id, + 0, + 1, + )?; + amber_site_controller::write_control_state(&state_path, &controller_state)?; + write_json_file( + &crate::mixed_run::desired_links_path(&site_state_root), + &crate::mixed_run::DesiredLinkState { + schema: crate::mixed_run::DESIRED_LINKS_SCHEMA.to_string(), + version: crate::mixed_run::DESIRED_LINKS_VERSION, + ..Default::default() + }, + )?; + + let listen_addr = SocketAddr::from(([0, 0, 0, 0], controller_port)); + let authority_url = format!( + "http://{}:{}", + amber_site_controller::SITE_CONTROLLER_SERVICE_NAME, + controller_port + ); + let local_router_control = + crate::mixed_run::site_controller_local_router_control(export.kind, &artifact_root); + let control_state_auth_token = + standalone_control_state_auth_token(&run_plan.mesh_scope, run_id, "site-controller"); + let router_mesh_port = Some(24000); + if let Some(port) = router_mesh_port + && export.kind == SiteKind::Compose + { + amber_site_controller::set_compose_router_published_mesh_port(&artifact_root, port)?; + } + let compose_project = (export.kind == SiteKind::Compose) + .then(|| crate::mixed_run::compose_project_name(run_id, &export.site_id)); + let controller_identity_path = + crate::mixed_run::site_controller_identity_path(export.kind, &artifact_root, None)?; + + let plan = amber_site_controller::write_site_controller_plan( + &plan_path, + run_id, + &run_plan.mesh_scope, + &export.site_id, + export.kind, + listen_addr, + &authority_url, + &site_plan.router_identity_id, + &BTreeMap::new(), + &BTreeMap::new(), + &BTreeMap::new(), + Some(local_router_control.as_str()), + None, + None, + None, + &plan_state_path, + &plan_run_root, + &plan_state_root, + &plan_site_state_root, + &plan_artifact_root, + &control_state_auth_token, + controller_identity_path.as_deref(), + None, + None, + router_mesh_port, + compose_project.as_deref(), + kubernetes_namespace.as_deref(), + None, + None, + &launch_env, + )?; + + let controller_image = crate::mixed_run::site_controller_image_reference()?; + match export.kind { + SiteKind::Compose => { + let plan_mount_source = + compose_mount_source_relative_to_artifact(&artifact_root, &plan_path)?; + amber_site_controller::inject_compose_site_controller_with_mount_sources( + &artifact_root, + &plan, + ".", + &plan_mount_source, + &controller_image, + )?; + set_compose_environment_values(&artifact_root, &launch_env)?; + } + SiteKind::Kubernetes => { + amber_site_controller::inject_kubernetes_site_controller( + &artifact_root, + &plan, + &controller_image, + )?; + let _ = std::fs::remove_dir_all(&state_root); + } + SiteKind::Direct | SiteKind::Vm => {} + } + + Ok(()) +} + +fn compose_mount_source_relative_to_artifact(artifact_root: &Path, path: &Path) -> Result { + let relative = path + .strip_prefix(artifact_root) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to express {} as a path relative to {}", + path.display(), + artifact_root.display() + ) + })?; + let relative = relative + .iter() + .map(|part| part.to_string_lossy()) + .collect::>() + .join("/"); + Ok(format!("./{relative}")) +} + +fn standalone_control_state_auth_token(mesh_scope: &str, run_id: &str, purpose: &str) -> String { + base64::engine::general_purpose::STANDARD.encode( + amber_mesh::MeshIdentity::derive( + format!("/framework/{purpose}"), + Some(mesh_scope.to_string()), + &format!("standalone-control-state:{run_id}"), + ) + .public_key, ) } +fn standalone_controller_temp_state_root(site_id: &str) -> Result { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .into_diagnostic() + .wrap_err("system clock is before the Unix epoch")? + .as_nanos(); + let mut safe_site = String::new(); + for ch in site_id.chars() { + safe_site.push(if ch.is_ascii_alphanumeric() { ch } else { '-' }); + } + Ok(std::env::temp_dir().join(format!( + "amber-standalone-controller-{safe_site}-{}-{nanos}", + std::process::id() + ))) +} + +fn write_json_file(path: &Path, value: &T) -> Result<()> { + let json = serde_json::to_vec_pretty(value) + .map_err(|err| miette::miette!("failed to serialize {}: {err}", path.display()))?; + write_artifact(path, &json) +} + +fn set_compose_environment_values( + artifact_root: &Path, + values: &BTreeMap, +) -> Result<()> { + let compose_path = artifact_root.join("compose.yaml"); + let raw = fs::read_to_string(&compose_path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", compose_path.display()))?; + let mut document: serde_yaml::Value = serde_yaml::from_str(&raw) + .into_diagnostic() + .wrap_err_with(|| format!("failed to parse {}", compose_path.display()))?; + let services = document + .as_mapping_mut() + .and_then(|root| root.get_mut(yaml_string("services"))) + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or_else(|| { + miette::miette!( + "compose file {} is missing services", + compose_path.display() + ) + })?; + for service in services.values_mut() { + let service = service.as_mapping_mut().ok_or_else(|| { + miette::miette!( + "compose service in {} is not a mapping", + compose_path.display() + ) + })?; + let environment = service + .entry(yaml_string("environment")) + .or_insert_with(|| serde_yaml::Value::Mapping(serde_yaml::Mapping::new())); + set_compose_environment_entries(environment, values, &compose_path)?; + } + let rendered = serde_yaml::to_string(&document) + .into_diagnostic() + .wrap_err_with(|| format!("failed to serialize {}", compose_path.display()))?; + fs::write(&compose_path, rendered) + .into_diagnostic() + .wrap_err_with(|| format!("failed to write {}", compose_path.display())) +} + +fn set_compose_environment_entries( + environment: &mut serde_yaml::Value, + values: &BTreeMap, + compose_path: &Path, +) -> Result<()> { + match environment { + serde_yaml::Value::Mapping(map) => { + for (name, value) in values { + map.insert(yaml_string(name), yaml_string(value)); + } + Ok(()) + } + serde_yaml::Value::Sequence(entries) => { + for (name, value) in values { + let rendered = format!("{name}={value}"); + let mut found = false; + for entry in entries.iter_mut() { + let Some(raw) = entry.as_str() else { + continue; + }; + if raw == name + || raw + .strip_prefix(name) + .is_some_and(|tail| tail.starts_with('=')) + { + *entry = yaml_string(&rendered); + found = true; + } + } + if !found { + entries.push(yaml_string(&rendered)); + } + } + Ok(()) + } + serde_yaml::Value::Null => { + *environment = serde_yaml::Value::Mapping(serde_yaml::Mapping::new()); + set_compose_environment_entries(environment, values, compose_path) + } + _ => Err(miette::miette!( + "compose file {} has an unsupported environment shape", + compose_path.display() + )), + } +} + +fn yaml_string(value: &str) -> serde_yaml::Value { + serde_yaml::Value::String(value.to_string()) +} + pub(crate) fn write_directory_output( root: &Path, label: &str, diff --git a/cli/src/direct_runtime/preview.rs b/cli/src/direct_runtime/preview.rs index aa350e23..1c2cd073 100644 --- a/cli/src/direct_runtime/preview.rs +++ b/cli/src/direct_runtime/preview.rs @@ -375,7 +375,12 @@ pub(crate) fn build_direct_site_launch_preview( &direct_plan.runtime_addresses, &runtime_state, )?; - spec.hidden_paths.push(runtime_root.join("mesh")); + if !matches!( + component.program.execution, + DirectProgramExecutionPlan::InternalSiteController + ) { + spec.hidden_paths.push(runtime_root.join("mesh")); + } let resolved_process = match &component.program.execution { DirectProgramExecutionPlan::Direct { .. } => None, DirectProgramExecutionPlan::HelperRunner { .. } => { diff --git a/cli/src/direct_runtime/runtime.rs b/cli/src/direct_runtime/runtime.rs index 48b2e9f9..5b49269c 100644 --- a/cli/src/direct_runtime/runtime.rs +++ b/cli/src/direct_runtime/runtime.rs @@ -492,7 +492,12 @@ pub(crate) async fn run_direct_init(args: RunDirectInitArgs) -> Result<()> { &direct_plan.runtime_addresses, &runtime_state, )?; - spec.hidden_paths.push(runtime_root.join("mesh")); + if !matches!( + component.program.execution, + DirectProgramExecutionPlan::InternalSiteController + ) { + spec.hidden_paths.push(runtime_root.join("mesh")); + } #[cfg(target_os = "linux")] { if direct_component_program_joins_sidecar(component) { diff --git a/cli/src/mixed_run.rs b/cli/src/mixed_run.rs index 46c714ea..b36e8db0 100644 --- a/cli/src/mixed_run.rs +++ b/cli/src/mixed_run.rs @@ -66,8 +66,8 @@ const SITE_STATE_SCHEMA: &str = "amber.run.site_state"; const SITE_STATE_VERSION: u32 = 3; const SITE_PLAN_SCHEMA: &str = "amber.run.site_supervisor_plan"; const SITE_PLAN_VERSION: u32 = 2; -const DESIRED_LINKS_SCHEMA: &str = "amber.run.desired_links"; -const DESIRED_LINKS_VERSION: u32 = 1; +pub(crate) const DESIRED_LINKS_SCHEMA: &str = "amber.run.desired_links"; +pub(crate) const DESIRED_LINKS_VERSION: u32 = 1; const OTLP_SINK_PLAN_SCHEMA: &str = "amber.run.observability_sink"; const OTLP_SINK_PLAN_VERSION: u32 = 1; const OTELCOL_UPSTREAM_ENV: &str = "AMBER_OTEL_UPSTREAM_OTLP_HTTP_ENDPOINT"; diff --git a/cli/src/mixed_run/launch_bundle.rs b/cli/src/mixed_run/launch_bundle.rs index 1dbccd93..0c86b3ae 100644 --- a/cli/src/mixed_run/launch_bundle.rs +++ b/cli/src/mixed_run/launch_bundle.rs @@ -9,14 +9,14 @@ pub(super) fn site_controller_image_reference_from_overrides( .unwrap_or_else(|| amber_images::AMBER_SITE_CONTROLLER.reference.to_string()) } -fn site_controller_image_reference() -> Result { +pub(crate) fn site_controller_image_reference() -> Result { let overrides = amber_images::parse_dev_image_tag_overrides(amber_images::INTERNAL_IMAGE_OVERRIDE_KEYS) .map_err(|err| miette::miette!(err))?; Ok(site_controller_image_reference_from_overrides(&overrides)) } -pub(super) fn site_controller_local_router_control(kind: SiteKind, artifact_dir: &Path) -> String { +pub(crate) fn site_controller_local_router_control(kind: SiteKind, artifact_dir: &Path) -> String { match kind { SiteKind::Direct => format!( "unix://{}", @@ -31,7 +31,7 @@ pub(super) fn site_controller_local_router_control(kind: SiteKind, artifact_dir: } } -fn site_controller_component_port(site_plan: &RunSitePlan) -> Result> { +pub(crate) fn site_controller_component_port(site_plan: &RunSitePlan) -> Result> { let scenario = amber_scenario::Scenario::try_from(site_plan.scenario_ir.clone()).map_err(|err| { miette::miette!( @@ -79,7 +79,50 @@ fn site_controller_component_port(site_plan: &RunSitePlan) -> Result } } -pub(super) fn prepare_site_state_root(site_state_root: &Path, kind: SiteKind) -> Result<()> { +pub(crate) fn site_controller_identity_path( + kind: SiteKind, + artifact_dir: &Path, + runtime_root: Option<&str>, +) -> Result> { + match kind { + SiteKind::Compose | SiteKind::Kubernetes => Ok(Some( + amber_site_controller::SITE_CONTROLLER_MESH_IDENTITY_PATH.to_string(), + )), + SiteKind::Direct => { + let Some(runtime_root) = runtime_root else { + return Ok(None); + }; + let direct_plan_path = + artifact_dir.join(amber_compiler::reporter::direct::DIRECT_PLAN_FILENAME); + let direct_plan: amber_compiler::reporter::direct::DirectPlan = + read_json(&direct_plan_path, "direct plan")?; + let controller = direct_plan + .components + .iter() + .find(|component| { + matches!( + component.program.execution, + amber_compiler::reporter::direct::DirectProgramExecutionPlan::InternalSiteController + ) + }) + .ok_or_else(|| { + miette::miette!( + "direct site controller plan is missing its internal site controller \ + component" + ) + })?; + Ok(Some( + Path::new(runtime_root) + .join(&controller.sidecar.mesh_identity_path) + .display() + .to_string(), + )) + } + SiteKind::Vm => Ok(None), + } +} + +pub(crate) fn prepare_site_state_root(site_state_root: &Path, kind: SiteKind) -> Result<()> { fs::create_dir_all(site_state_root) .into_diagnostic() .wrap_err_with(|| { @@ -174,17 +217,6 @@ pub(super) fn materialize_launch_bundle( &run_plan.mesh_scope, "site-controller", ); - let dynamic_capability_signing_seed_b64 = amber_mesh::dynamic_caps::signing_seed_b64( - &amber_mesh::dynamic_caps::signing_key_from_seed( - amber_mesh::dynamic_caps::generate_dynamic_capability_signing_seed(), - ), - ); - let dynamic_caps_token_verify_key_b64 = amber_mesh::dynamic_caps::verify_key_b64( - &amber_mesh::dynamic_caps::signing_key_from_seed_b64(&dynamic_capability_signing_seed_b64) - .map_err(|err| { - miette::miette!("site controller dynamic capability signing seed is invalid: {err}") - })?, - ); let observability = materialize_observability(bundle_root, run_id, &run_plan.mesh_scope, observability)?; let observability_endpoint = observability @@ -223,11 +255,7 @@ pub(super) fn materialize_launch_bundle( url, } }); - let mut framework_env = BTreeMap::new(); - framework_env.insert( - amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV.to_string(), - dynamic_caps_token_verify_key_b64.clone(), - ); + let framework_env = BTreeMap::new(); patch_site_artifacts( &artifact_dir, run_id, @@ -275,7 +303,6 @@ pub(super) fn materialize_launch_bundle( site_id, site_index, run_plan.sites.len(), - &dynamic_capability_signing_seed_b64, )?; amber_site_controller::write_control_state(&controller.state_path, &controller_state)?; } @@ -434,7 +461,12 @@ pub(super) fn materialize_launch_bundle( &site.site_state_root, &site.artifact_dir, &control_state_auth_token, - &dynamic_caps_token_verify_key_b64, + site_controller_identity_path( + site.site_plan.site.kind, + &site.artifact_dir, + site.base_supervisor_plan.runtime_root.as_deref(), + )? + .as_deref(), site.base_supervisor_plan.storage_root.as_deref(), site.base_supervisor_plan.runtime_root.as_deref(), site.base_supervisor_plan.router_mesh_port, diff --git a/cli/src/mixed_run/supervisor.rs b/cli/src/mixed_run/supervisor.rs index e546a4d5..11eab3f3 100644 --- a/cli/src/mixed_run/supervisor.rs +++ b/cli/src/mixed_run/supervisor.rs @@ -1833,7 +1833,7 @@ fn wait_for_named_kubernetes_resource( } } -pub(super) fn compose_project_name(run_id: &str, site_id: &str) -> String { +pub(crate) fn compose_project_name(run_id: &str, site_id: &str) -> String { let mut out = String::from("amber_"); for ch in format!("{run_id}_{site_id}").chars() { if ch.is_ascii_alphanumeric() || ch == '_' || ch == '-' { diff --git a/compiler/src/targets/mesh/docker_compose/mod.rs b/compiler/src/targets/mesh/docker_compose/mod.rs index 5aa1290c..73ccc92b 100644 --- a/compiler/src/targets/mesh/docker_compose/mod.rs +++ b/compiler/src/targets/mesh/docker_compose/mod.rs @@ -876,11 +876,11 @@ fn render_docker_compose_inner( // depends_on: own sidecar + strong deps provider programs (+ amber-init for helper-backed services) let program_plan = program_plans.get(id).expect("program plan computed"); + let is_site_controller = + framework_component_controller_metadata(s.component(*id).metadata.as_ref()).is_some(); let image = if Some(*id) == docker_gateway_component { images.docker_gateway.clone() - } else if framework_component_controller_metadata(s.component(*id).metadata.as_ref()) - .is_some() - { + } else if is_site_controller { images.site_controller.clone() } else { let image_plan = program_plan.image().ok_or_else(|| { @@ -1038,6 +1038,11 @@ fn render_docker_compose_inner( storage_mount.mount_path )); } + if is_site_controller { + program_service + .volumes + .push(format!("{sidecar_volume}:{MESH_CONFIG_DIR}:ro")); + } configure_program_log_shipping( &mut program_service, &label, diff --git a/compiler/src/targets/mesh/kubernetes/mod.rs b/compiler/src/targets/mesh/kubernetes/mod.rs index e0ae365b..f4f507b7 100644 --- a/compiler/src/targets/mesh/kubernetes/mod.rs +++ b/compiler/src/targets/mesh/kubernetes/mod.rs @@ -913,6 +913,13 @@ pub(crate) fn emit_kubernetes_artifact_with_options( MESH_SECRET_VOLUME_NAME.to_string(), mesh_secret, )); + if is_site_controller { + container.volume_mounts.push(VolumeMount { + name: MESH_SECRET_VOLUME_NAME.to_string(), + mount_path: MESH_CONFIG_DIR.to_string(), + read_only: Some(true), + }); + } let mut sidecar_env = vec![ EnvVar::literal( diff --git a/compiler/src/targets/mesh/kubernetes/tests/rendering.rs b/compiler/src/targets/mesh/kubernetes/tests/rendering.rs index 73d33469..56f97768 100644 --- a/compiler/src/targets/mesh/kubernetes/tests/rendering.rs +++ b/compiler/src/targets/mesh/kubernetes/tests/rendering.rs @@ -787,11 +787,6 @@ fn kubernetes_templates_omit_dynamic_caps_sidecar_control_env_without_local_cont !sidecar_env.contains("AMBER_FRAMEWORK_COMPONENT_CONTROLLER_AUTH_TOKEN="), "controller auth must not be injected as ambient sidecar env: {sidecar_env}" ); - assert!( - !sidecar_env.contains("AMBER_DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64="), - "dynamic caps verify keys must not be injected when no local controller exists: \ - {sidecar_env}" - ); } let kustomization = artifact diff --git a/compiler/src/targets/mesh/mesh_config.rs b/compiler/src/targets/mesh/mesh_config.rs index 1582bc8d..66226be3 100644 --- a/compiler/src/targets/mesh/mesh_config.rs +++ b/compiler/src/targets/mesh/mesh_config.rs @@ -2,10 +2,9 @@ use std::collections::{BTreeSet, HashMap}; use amber_manifest::NetworkProtocol; use amber_mesh::{ - DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, InboundRoute, InboundTarget, MeshConfigTemplate, - MeshIdentityTemplate, MeshPeerTemplate, MeshProtocol, OutboundRoute, component_route_id, - http_route_plugins_for_capability_kind, router_export_route_id, router_external_route_id, - telemetry::SCENARIO_RUN_ID_ENV, + InboundRoute, InboundTarget, MeshConfigTemplate, MeshIdentityTemplate, MeshPeerTemplate, + MeshProtocol, OutboundRoute, component_route_id, http_route_plugins_for_capability_kind, + router_export_route_id, router_external_route_id, telemetry::SCENARIO_RUN_ID_ENV, }; use amber_scenario::{ComponentId, Scenario}; use base64::Engine as _; @@ -200,12 +199,6 @@ pub(crate) fn build_mesh_config_plan( let mut component_configs: HashMap = HashMap::new(); let mut component_sidecar_env_passthrough = Vec::new(); push_env_passthrough_once(&mut component_sidecar_env_passthrough, SCENARIO_RUN_ID_ENV); - if local_controller_component.is_some() { - push_env_passthrough_once( - &mut component_sidecar_env_passthrough, - DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, - ); - } for &id in mesh_plan.program_components() { let identity = identities_by_component .get(&id) diff --git a/runtime/mesh/src/dynamic_caps.rs b/runtime/mesh/src/dynamic_caps.rs index 4651182e..c8071ef2 100644 --- a/runtime/mesh/src/dynamic_caps.rs +++ b/runtime/mesh/src/dynamic_caps.rs @@ -1,11 +1,7 @@ use std::fmt; -use base64::{ - Engine as _, - engine::general_purpose::{STANDARD, URL_SAFE_NO_PAD}, -}; +use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use ed25519_dalek::{Signature, Signer as _, SigningKey, Verifier as _, VerifyingKey}; -use rand_core::OsRng; use serde::{Deserialize, Serialize}; use url::Url; @@ -18,7 +14,7 @@ pub const DYNAMIC_CAPS_REF_VERSION: u32 = 1; pub const DYNAMIC_CAPS_API_URL_ENV: &str = "AMBER_DYNAMIC_CAPS_API_URL"; pub const FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME: &str = "__amber_internal_site_controller"; -pub const DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV: &str = "AMBER_DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64"; +const DYNAMIC_CAPABILITY_REF_SIGNATURE_DOMAIN: &[u8] = b"amber.dynamic-capability-ref.v1\0"; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -246,60 +242,13 @@ pub struct GrantSnapshotIr { pub struct DynamicCapabilityRefClaims { pub version: u32, pub run_id: String, + pub issuer_id: String, pub grant_id: String, pub holder_component_id: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub descriptor_hint: Option, } -pub fn generate_dynamic_capability_signing_seed() -> [u8; 32] { - SigningKey::generate(&mut OsRng).to_bytes() -} - -pub fn signing_key_from_seed(seed: [u8; 32]) -> SigningKey { - SigningKey::from_bytes(&seed) -} - -pub fn signing_key_from_seed_b64(raw: &str) -> Result { - let decoded = STANDARD.decode(raw.as_bytes()).map_err(|err| { - DynamicCapabilityRefError::InvalidToken(format!( - "failed to decode dynamic capability signing seed: {err}" - )) - })?; - let seed: [u8; 32] = decoded.as_slice().try_into().map_err(|_| { - DynamicCapabilityRefError::InvalidToken( - "dynamic capability signing seed must be exactly 32 bytes".to_string(), - ) - })?; - Ok(signing_key_from_seed(seed)) -} - -pub fn signing_seed_b64(signing_key: &SigningKey) -> String { - STANDARD.encode(signing_key.to_bytes()) -} - -pub fn verify_key_b64(signing_key: &SigningKey) -> String { - STANDARD.encode(signing_key.verifying_key().to_bytes()) -} - -pub fn verify_key_from_b64(raw: &str) -> Result { - let decoded = STANDARD.decode(raw.as_bytes()).map_err(|err| { - DynamicCapabilityRefError::InvalidToken(format!( - "failed to decode dynamic capability verify key: {err}" - )) - })?; - let bytes: [u8; 32] = decoded.as_slice().try_into().map_err(|_| { - DynamicCapabilityRefError::InvalidToken( - "dynamic capability verify key must be exactly 32 bytes".to_string(), - ) - })?; - VerifyingKey::from_bytes(&bytes).map_err(|err| { - DynamicCapabilityRefError::InvalidToken(format!( - "dynamic capability verify key is malformed: {err}" - )) - }) -} - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ExactRootRouteError { InvalidLogicalComponentId, @@ -405,11 +354,7 @@ pub fn encode_dynamic_capability_ref( claims: DynamicCapabilityRefClaims, signing_key: &SigningKey, ) -> Result { - let payload = serde_json::to_vec(&claims).map_err(|err| { - DynamicCapabilityRefError::InvalidToken(format!( - "failed to encode dynamic capability ref claims: {err}" - )) - })?; + let payload = dynamic_capability_ref_signature_payload(&claims)?; let signature = signing_key.sign(&payload); let token = DynamicCapabilityRefToken { claims, @@ -511,16 +456,27 @@ pub fn verify_dynamic_capability_ref( parsed: &ParsedDynamicCapabilityRef, verify_key: &VerifyingKey, ) -> Result<(), DynamicCapabilityRefError> { - let claims_bytes = serde_json::to_vec(&parsed.claims).map_err(|err| { - DynamicCapabilityRefError::InvalidToken(format!( - "failed to re-encode dynamic capability ref claims: {err}" - )) - })?; + let claims_bytes = dynamic_capability_ref_signature_payload(&parsed.claims)?; verify_key .verify(&claims_bytes, &parsed.signature) .map_err(|_| DynamicCapabilityRefError::InvalidSignature) } +fn dynamic_capability_ref_signature_payload( + claims: &DynamicCapabilityRefClaims, +) -> Result, DynamicCapabilityRefError> { + let claims_json = serde_json::to_vec(claims).map_err(|err| { + DynamicCapabilityRefError::InvalidToken(format!( + "failed to encode dynamic capability ref claims: {err}" + )) + })?; + let mut payload = + Vec::with_capacity(DYNAMIC_CAPABILITY_REF_SIGNATURE_DOMAIN.len() + claims_json.len()); + payload.extend_from_slice(DYNAMIC_CAPABILITY_REF_SIGNATURE_DOMAIN); + payload.extend_from_slice(&claims_json); + Ok(payload) +} + pub fn build_dynamic_capability_ref_url( claims: DynamicCapabilityRefClaims, signing_key: &SigningKey, @@ -552,6 +508,8 @@ pub fn build_dynamic_capability_ref_url( #[cfg(test)] mod tests { + use rand_core::OsRng; + use super::*; use crate::HttpRoutePlugin; @@ -581,6 +539,7 @@ mod tests { DynamicCapabilityRefClaims { version: DYNAMIC_CAPS_REF_VERSION, run_id: "run-123".to_string(), + issuer_id: "/controller".to_string(), grant_id: "g_abc".to_string(), holder_component_id: "components./worker".to_string(), descriptor_hint: Some("worker.http".to_string()), @@ -589,7 +548,7 @@ mod tests { #[test] fn dynamic_capability_ref_round_trip_preserves_claims_and_suffix() { - let signing_key = signing_key_from_seed(generate_dynamic_capability_signing_seed()); + let signing_key = SigningKey::generate(&mut OsRng); let raw = build_dynamic_capability_ref_url( test_claims(), &signing_key, @@ -609,7 +568,7 @@ mod tests { #[test] fn dynamic_capability_ref_rejects_signature_tampering() { - let signing_key = signing_key_from_seed(generate_dynamic_capability_signing_seed()); + let signing_key = SigningKey::generate(&mut OsRng); let raw = build_dynamic_capability_ref_url(test_claims(), &signing_key, "/", None, None) .expect("dynamic ref should build"); let mut parsed = Url::parse(&raw).expect("dynamic ref url should parse"); @@ -630,14 +589,6 @@ mod tests { assert!(matches!(err, DynamicCapabilityRefError::InvalidSignature)); } - #[test] - fn verify_key_b64_round_trip_preserves_key_material() { - let signing_key = signing_key_from_seed(generate_dynamic_capability_signing_seed()); - let encoded = verify_key_b64(&signing_key); - let decoded = verify_key_from_b64(&encoded).expect("verify key should decode"); - assert_eq!(decoded.to_bytes(), signing_key.verifying_key().to_bytes()); - } - #[test] fn held_entry_summary_serialization_keeps_empty_materializations() { let summary = HeldEntrySummary { diff --git a/runtime/mesh/src/lib.rs b/runtime/mesh/src/lib.rs index ce378e5f..e0ed924d 100644 --- a/runtime/mesh/src/lib.rs +++ b/runtime/mesh/src/lib.rs @@ -209,8 +209,6 @@ pub const MESH_IDENTITY_FILENAME: &str = "mesh-identity.json"; pub const FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME: &str = dynamic_caps::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME; pub const DYNAMIC_CAPS_API_URL_ENV: &str = dynamic_caps::DYNAMIC_CAPS_API_URL_ENV; -pub const DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV: &str = - dynamic_caps::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV; fn stable_temp_socket_root() -> PathBuf { #[cfg(unix)] diff --git a/runtime/router/src/dynamic_caps.rs b/runtime/router/src/dynamic_caps.rs index b4c3e6c5..78fef51e 100644 --- a/runtime/router/src/dynamic_caps.rs +++ b/runtime/router/src/dynamic_caps.rs @@ -1,7 +1,7 @@ -use std::{collections::BTreeMap, env, sync::Arc}; +use std::{collections::BTreeMap, sync::Arc}; use amber_mesh::{ - MeshProtocol, + MeshProtocol, OutboundRoute, component_protocol::{ProtocolErrorCode, ProtocolErrorResponse}, dynamic_caps::{ self as mesh_dynamic_caps, DescriptorIr, HeldEntryDetail, HeldEntryKind, HeldEntryState, @@ -10,7 +10,6 @@ use amber_mesh::{ MaterializedHandleSummary, RevokeRequest, RevokeResponse, RootAuthoritySelectorIr, ShareRequest, ShareResponse, ShareSource, }, - telemetry::SCENARIO_RUN_ID_ENV, }; use base64::Engine as _; use http_body_util::{BodyExt as _, Full}; @@ -25,6 +24,7 @@ const DYNAMIC_CAPS_CONTROLLER_HELD_DETAIL_PATH: &str = "/v1/controller/dynamic-c const DYNAMIC_CAPS_CONTROLLER_SHARE_PATH: &str = "/v1/controller/dynamic-caps/share"; const DYNAMIC_CAPS_CONTROLLER_INSPECT_REF_PATH: &str = "/v1/controller/dynamic-caps/inspect-ref"; const DYNAMIC_CAPS_CONTROLLER_REVOKE_PATH: &str = "/v1/controller/dynamic-caps/revoke"; +const DYNAMIC_CAPS_CONTROLLER_RESOLVE_REF_PATH: &str = "/v1/controller/dynamic-caps/resolve-ref"; const DYNAMIC_CAPS_CONTROLLER_RESOLVE_ORIGIN_PATH: &str = "/v1/controller/dynamic-caps/resolve-origin"; const DYNAMIC_CAPS_HANDLE_PREFIX: &str = "/v1/handles/"; @@ -34,8 +34,6 @@ const DYNAMIC_CAPS_WATCH_POLL_INTERVAL: Duration = Duration::from_millis(250); #[derive(Debug)] struct DynamicCapsControllerEnv { control_url: String, - verify_key_raw: String, - run_id: String, } #[derive(Clone, Debug, Serialize)] @@ -92,6 +90,12 @@ struct ControlDynamicResolveOriginRequest { source: DynamicCapabilityControlSourceRequest, } +#[derive(Clone, Debug, Serialize)] +struct ControlDynamicResolveRefRequest { + holder_component_id: String, + r#ref: String, +} + #[derive(Clone, Debug, Deserialize)] struct ControlDynamicResolveOriginResponse { held_id: String, @@ -104,13 +108,22 @@ struct ControlDynamicResolveOriginResponse { origin_peer_addr: String, } +#[derive(Clone, Debug, Deserialize)] +struct ControlDynamicResolveRefResponse { + #[serde(flatten)] + origin: ControlDynamicResolveOriginResponse, + relative_path: String, + #[serde(default)] + query: Option, + #[serde(default)] + fragment: Option, +} + #[derive(Clone)] pub(super) struct DynamicCapsRuntime { listen_addr: SocketAddr, component_id: Arc, - run_id: Arc, control_url: Arc, - verify_key: ed25519_dalek::VerifyingKey, config: Arc, client: Arc, a2a_url_rewrite_table: Arc, @@ -227,14 +240,10 @@ impl DynamicCapsRuntime { ); return Ok(None); }; - let verify_key = mesh_dynamic_caps::verify_key_from_b64(&control_env.verify_key_raw) - .map_err(|err| RouterError::InvalidConfig(err.to_string()))?; Ok(Some(Arc::new(Self { listen_addr, component_id: Arc::::from(format!("components.{}", config.identity.id)), - run_id: Arc::::from(control_env.run_id), control_url: Arc::::from(control_env.control_url), - verify_key, config, client, a2a_url_rewrite_table, @@ -360,6 +369,34 @@ impl DynamicCapsRuntime { .await } + async fn control_inspect_ref( + &self, + raw_ref: &str, + ) -> Result { + self.control_post_json( + DYNAMIC_CAPS_CONTROLLER_INSPECT_REF_PATH, + &ControlDynamicInspectRefRequest { + holder_component_id: self.component_id.to_string(), + r#ref: raw_ref.to_string(), + }, + ) + .await + } + + async fn control_resolve_ref( + &self, + raw_ref: &str, + ) -> Result { + self.control_post_json( + DYNAMIC_CAPS_CONTROLLER_RESOLVE_REF_PATH, + &ControlDynamicResolveRefRequest { + holder_component_id: self.component_id.to_string(), + r#ref: raw_ref.to_string(), + }, + ) + .await + } + fn next_handle_id(state: &mut DynamicCapsLocalState) -> String { let handle_id = format!( "{DYNAMIC_CAPS_HANDLE_ID_PREFIX}{:016x}", @@ -386,38 +423,6 @@ impl DynamicCapsRuntime { Ok(url.to_string()) } - fn parse_ref_for_local_holder( - &self, - raw_ref: &str, - ) -> Result { - let parsed = mesh_dynamic_caps::decode_dynamic_capability_ref_unverified(raw_ref) - .map_err(|err| self.protocol_error(ProtocolErrorCode::MalformedRef, err.to_string()))?; - if parsed.claims.version != mesh_dynamic_caps::DYNAMIC_CAPS_REF_VERSION { - return Err(self.protocol_error( - ProtocolErrorCode::MalformedRef, - format!( - "dynamic capability ref version {} is unsupported", - parsed.claims.version - ), - )); - } - if parsed.claims.run_id != self.run_id.as_ref() { - return Err(self.protocol_error( - ProtocolErrorCode::MalformedRef, - "dynamic capability ref belongs to a different run", - )); - } - if parsed.claims.holder_component_id != self.component_id.as_ref() { - return Err(self.protocol_error( - ProtocolErrorCode::RecipientMismatch, - "dynamic capability ref is bound to a different holder", - )); - } - mesh_dynamic_caps::verify_dynamic_capability_ref(&parsed, &self.verify_key) - .map_err(|err| self.protocol_error(ProtocolErrorCode::MalformedRef, err.to_string()))?; - Ok(parsed) - } - fn source_from_held_detail( &self, detail: &HeldEntryDetail, @@ -654,6 +659,15 @@ impl DynamicCapsRuntime { descriptor_hint: Option, ) -> Result { let resolved = self.control_resolve_origin(source).await?; + self.ensure_resolved_dynamic_handle_materialized(resolved, descriptor_hint) + .await + } + + async fn ensure_resolved_dynamic_handle_materialized( + &self, + resolved: ControlDynamicResolveOriginResponse, + descriptor_hint: Option, + ) -> Result { if resolved.origin_protocol != "http" { return Err(self.protocol_error( ProtocolErrorCode::PathEstablishmentFailed, @@ -816,23 +830,18 @@ impl DynamicCapsRuntime { request: MaterializeRequest, ) -> Result { if let Some(raw_ref) = request.r#ref.as_deref() { - let parsed = self.parse_ref_for_local_holder(raw_ref)?; + let resolved = self.control_resolve_ref(raw_ref).await?; let materialized = self - .ensure_dynamic_handle_materialized( - DynamicCapabilityControlSourceRequest::Grant { - grant_id: parsed.claims.grant_id.clone(), - }, - None, - ) + .ensure_resolved_dynamic_handle_materialized(resolved.origin, None) .await?; return Ok(MaterializeResponse { held_id: materialized.held_id, handle_id: materialized.handle_id.clone(), url: self.materialized_handle_url( &materialized.handle_id, - &parsed.relative_path, - parsed.query.as_deref(), - parsed.fragment.as_deref(), + &resolved.relative_path, + resolved.query.as_deref(), + resolved.fragment.as_deref(), )?, }); } @@ -1096,36 +1105,13 @@ impl DynamicCapsRuntime { } } -fn nonempty_env_var(name: &str) -> Option { - env::var(name) - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) -} - -fn required_dynamic_caps_env_var( - name: &'static str, - value: Option, -) -> Result { - value.ok_or_else(|| { - RouterError::InvalidConfig(format!( - "{name} must be set when dynamic_caps_listen is configured" - )) +fn local_dynamic_caps_controller_route(config: &MeshConfig) -> Option<&OutboundRoute> { + config.outbound.iter().find(|route| { + route.protocol == MeshProtocol::Http + && route.capability == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME }) } -fn local_dynamic_caps_controller_url(config: &MeshConfig) -> Option { - config - .outbound - .iter() - .find(|route| { - route.protocol == MeshProtocol::Http - && route.capability - == amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME - }) - .map(|route| format!("http://127.0.0.1:{}", route.listen_port)) -} - fn resolve_dynamic_caps_controller_env( config: &MeshConfig, ) -> Result, RouterError> { @@ -1133,70 +1119,21 @@ fn resolve_dynamic_caps_controller_env( return Ok(None); } - let Some(verify_key_raw) = nonempty_env_var(amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV) - else { - return Ok(None); - }; - - let local_control_url = local_dynamic_caps_controller_url(config); - let run_id = - required_dynamic_caps_env_var(SCENARIO_RUN_ID_ENV, nonempty_env_var(SCENARIO_RUN_ID_ENV))?; - let control_url = local_control_url.ok_or_else(|| { + let controller_route = local_dynamic_caps_controller_route(config); + let controller_route = controller_route.ok_or_else(|| { RouterError::InvalidConfig( "dynamic_caps_listen requires a local framework.component controller route".to_string(), ) })?; Ok(Some(DynamicCapsControllerEnv { - control_url, - verify_key_raw, - run_id, + control_url: format!("http://127.0.0.1:{}", controller_route.listen_port), })) } #[cfg(test)] mod tests { - use std::sync::{Mutex, OnceLock}; - use super::*; - static ENV_LOCK: OnceLock> = OnceLock::new(); - - struct EnvGuard { - saved: Vec<(&'static str, Option)>, - } - - impl EnvGuard { - fn replace(pairs: [(&'static str, Option<&str>); 2]) -> Self { - let saved = pairs - .iter() - .map(|(name, value)| { - let previous = env::var(name).ok(); - unsafe { - match value { - Some(value) => env::set_var(name, value), - None => env::remove_var(name), - } - } - (*name, previous) - }) - .collect(); - Self { saved } - } - } - - impl Drop for EnvGuard { - fn drop(&mut self) { - for (name, value) in self.saved.drain(..) { - unsafe { - match value { - Some(value) => env::set_var(name, value), - None => env::remove_var(name), - } - } - } - } - } - fn mesh_config_with_internal_controller_route() -> MeshConfig { MeshConfig { identity: amber_mesh::MeshIdentity::generate("/component/test", None), @@ -1237,39 +1174,8 @@ mod tests { config } - #[test] - fn resolve_dynamic_caps_controller_env_disables_listener_when_control_env_is_absent() { - let _guard = ENV_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("env lock"); - let _env = EnvGuard::replace([ - (amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, None), - (SCENARIO_RUN_ID_ENV, Some("run-1234")), - ]); - - assert!( - resolve_dynamic_caps_controller_env(&mesh_config_without_internal_controller_route()) - .expect("dynamic caps env should resolve") - .is_none(), - "sidecars without dynamic caps controller env should leave the listener disabled", - ); - } - #[test] fn resolve_dynamic_caps_controller_env_ignores_control_env_when_listener_is_disabled() { - let _guard = ENV_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("env lock"); - let _env = EnvGuard::replace([ - ( - amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, - Some("verify-key"), - ), - (SCENARIO_RUN_ID_ENV, Some("run-1234")), - ]); - assert!( resolve_dynamic_caps_controller_env(&mesh_config_without_dynamic_caps_listener()) .expect("dynamic caps env should resolve") @@ -1280,18 +1186,6 @@ mod tests { #[test] fn resolve_dynamic_caps_controller_env_rejects_missing_local_controller_route() { - let _guard = ENV_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("env lock"); - let _env = EnvGuard::replace([ - ( - amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, - Some("verify-key"), - ), - (SCENARIO_RUN_ID_ENV, Some("run-1234")), - ]); - let err = resolve_dynamic_caps_controller_env(&mesh_config_without_internal_controller_route()) .expect_err("dynamic caps without a local controller route must fail"); @@ -1307,18 +1201,6 @@ mod tests { #[test] fn resolve_dynamic_caps_controller_env_prefers_local_controller_route_without_auth_token() { - let _guard = ENV_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("env lock"); - let _env = EnvGuard::replace([ - ( - amber_mesh::DYNAMIC_CAPS_TOKEN_VERIFY_KEY_B64_ENV, - Some("verify-key"), - ), - (SCENARIO_RUN_ID_ENV, Some("run-1234")), - ]); - let config = mesh_config_with_internal_controller_route(); let env = resolve_dynamic_caps_controller_env(&config) .expect("dynamic caps env should resolve") @@ -1500,12 +1382,12 @@ async fn dynamic_caps_service( Err(err) => return Ok(protocol_response(&err)), } } else if let Some(raw_ref) = request.r#ref.as_deref() { - let parsed = match state.parse_ref_for_local_holder(raw_ref) { - Ok(parsed) => parsed, + let inspected = match state.control_inspect_ref(raw_ref).await { + Ok(inspected) => inspected, Err(err) => return Ok(protocol_response(&err)), }; DynamicCapabilityControlSourceRequest::Grant { - grant_id: parsed.claims.grant_id, + grant_id: inspected.grant_id, } } else { return Ok(protocol_response(&ProtocolErrorResponse { @@ -1539,9 +1421,6 @@ async fn dynamic_caps_service( Ok(request) => request, Err(response) => return Ok(response), }; - if let Err(err) = state.parse_ref_for_local_holder(&request.r#ref) { - return Ok(protocol_response(&err)); - } let response: InspectRefResponse = match state .control_post_json( DYNAMIC_CAPS_CONTROLLER_INSPECT_REF_PATH, diff --git a/runtime/site-controller/src/default_runtime.rs b/runtime/site-controller/src/default_runtime.rs index c41aa1af..a67f9dff 100644 --- a/runtime/site-controller/src/default_runtime.rs +++ b/runtime/site-controller/src/default_runtime.rs @@ -54,7 +54,9 @@ pub(crate) use self::child_runtime::{ pub(super) use self::site_artifacts::*; pub use self::{ child_runtime::cleanup_dynamic_site_children, - compose_controller::inject_compose_site_controller, + compose_controller::{ + inject_compose_site_controller, inject_compose_site_controller_with_mount_sources, + }, kubernetes_controller::inject_kubernetes_site_controller, site_artifacts::{ SiteControllerPeerRouterRoute, inject_site_controller_peer_router_routes, diff --git a/runtime/site-controller/src/default_runtime/child_runtime.rs b/runtime/site-controller/src/default_runtime/child_runtime.rs index 0dcc2b44..d73be888 100644 --- a/runtime/site-controller/src/default_runtime/child_runtime.rs +++ b/runtime/site-controller/src/default_runtime/child_runtime.rs @@ -1443,7 +1443,7 @@ mod tests { site_state_root: compose_state_root.display().to_string(), artifact_dir: compose_artifact.display().to_string(), control_state_auth_token: "test-auth".to_string(), - dynamic_caps_token_verify_key_b64: String::new(), + controller_identity_path: None, storage_root: None, runtime_root: None, router_mesh_port: Some(24034), @@ -1573,7 +1573,7 @@ mod tests { site_state_root: direct_state_root.display().to_string(), artifact_dir: direct_artifact.display().to_string(), control_state_auth_token: "test-auth".to_string(), - dynamic_caps_token_verify_key_b64: String::new(), + controller_identity_path: None, storage_root: Some( temp.path() .join("storage") diff --git a/runtime/site-controller/src/default_runtime/compose_controller.rs b/runtime/site-controller/src/default_runtime/compose_controller.rs index 5c61f2a1..e43926ca 100644 --- a/runtime/site-controller/src/default_runtime/compose_controller.rs +++ b/runtime/site-controller/src/default_runtime/compose_controller.rs @@ -141,6 +141,23 @@ pub fn inject_compose_site_controller( plan: &SiteControllerPlan, plan_path: &Path, controller_image: &str, +) -> Result<()> { + let plan_mount_source = plan_path.display().to_string(); + inject_compose_site_controller_with_mount_sources( + artifact_root, + plan, + &plan.run_root, + &plan_mount_source, + controller_image, + ) +} + +pub fn inject_compose_site_controller_with_mount_sources( + artifact_root: &Path, + plan: &SiteControllerPlan, + run_root_mount_source: &str, + plan_mount_source: &str, + controller_image: &str, ) -> Result<()> { let compose_path = artifact_root.join("compose.yaml"); let mut document = read_compose_document(&compose_path)?; @@ -205,8 +222,8 @@ pub fn inject_compose_site_controller( miette::miette!("compose site controller service has a non-sequence volumes field") })?; for volume in [ - format!("{}:{}", plan.run_root, plan.run_root), - format!("{}:{COMPOSE_CONTROLLER_PLAN_PATH}", plan_path.display()), + format!("{run_root_mount_source}:{}", plan.run_root), + format!("{plan_mount_source}:{COMPOSE_CONTROLLER_PLAN_PATH}"), format!("{COMPOSE_ROUTER_CONTROL_VOLUME_NAME}:{COMPOSE_ROUTER_CONTROL_SOCKET_DIR}"), format!("{DOCKER_SOCK_PATH}:{DOCKER_SOCK_PATH}"), ] { @@ -256,7 +273,7 @@ mod tests { site_state_root: run_root.join("site-state").display().to_string(), artifact_dir: run_root.join("artifact").display().to_string(), control_state_auth_token: "token".to_string(), - dynamic_caps_token_verify_key_b64: "verify".to_string(), + controller_identity_path: None, storage_root: None, runtime_root: None, router_mesh_port: Some(24000), diff --git a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs index 24c88dff..17e51ede 100644 --- a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs +++ b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs @@ -905,7 +905,7 @@ mod tests { site_state_root: root.join("site-state").display().to_string(), artifact_dir: root.join("artifact").display().to_string(), control_state_auth_token: "token".to_string(), - dynamic_caps_token_verify_key_b64: "verify".to_string(), + controller_identity_path: None, storage_root: None, runtime_root: None, router_mesh_port: Some(24000), diff --git a/runtime/site-controller/src/dynamic_caps.rs b/runtime/site-controller/src/dynamic_caps.rs index 4a229e7a..a9a0e35b 100644 --- a/runtime/site-controller/src/dynamic_caps.rs +++ b/runtime/site-controller/src/dynamic_caps.rs @@ -133,6 +133,12 @@ pub(crate) struct ControlDynamicInspectRefRequest { pub(crate) r#ref: String, } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ControlDynamicResolveRefRequest { + pub(crate) holder_component_id: String, + pub(crate) r#ref: String, +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct ControlDynamicRevokeRequest { pub(crate) caller_component_id: String, @@ -191,6 +197,17 @@ pub(crate) struct ControlDynamicResolveOriginResponse { pub(crate) origin_peer_addr: String, } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(crate) struct ControlDynamicResolveRefResponse { + #[serde(flatten)] + pub(crate) origin: ControlDynamicResolveOriginResponse, + pub(crate) relative_path: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) query: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) fragment: Option, +} + fn dynamic_grant_live_default() -> bool { true } @@ -1087,19 +1104,18 @@ pub(crate) fn mint_dynamic_capability_ref( state: &FrameworkControlState, grant: &DynamicGrantRecord, ) -> std::result::Result { - let signing_key = amber_mesh::dynamic_caps::signing_key_from_seed_b64( - &state.dynamic_capability_signing_seed_b64, - ) - .map_err(|err| { + let identity = dynamic_ref_issuer_identity(state)?; + let signing_key = identity.signing_key().map_err(|err| { protocol_error( ProtocolErrorCode::ControlStateUnavailable, - &format!("dynamic capability signing key is invalid: {err}"), + &format!("site controller mesh identity is invalid: {err}"), ) })?; amber_mesh::dynamic_caps::build_dynamic_capability_ref_url( amber_mesh::dynamic_caps::DynamicCapabilityRefClaims { version: amber_mesh::dynamic_caps::DYNAMIC_CAPS_REF_VERSION, run_id: state.run_id.clone(), + issuer_id: identity.id.clone(), grant_id: grant.grant_id.clone(), holder_component_id: grant.holder_component_id.clone(), descriptor_hint: Some(grant.descriptor.label.clone()), @@ -1117,20 +1133,22 @@ pub(crate) fn mint_dynamic_capability_ref( }) } +fn dynamic_ref_issuer_identity( + state: &FrameworkControlState, +) -> std::result::Result<&amber_mesh::MeshIdentitySecret, ProtocolErrorResponse> { + state.controller_identity.as_ref().ok_or_else(|| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + "site controller mesh identity is not loaded", + ) + }) +} + pub(crate) fn inspect_dynamic_ref( state: &FrameworkControlState, holder_component_id: &str, raw_ref: &str, ) -> std::result::Result { - let signing_key = amber_mesh::dynamic_caps::signing_key_from_seed_b64( - &state.dynamic_capability_signing_seed_b64, - ) - .map_err(|err| { - protocol_error( - ProtocolErrorCode::ControlStateUnavailable, - &format!("dynamic capability signing key is invalid: {err}"), - ) - })?; let parsed = amber_mesh::dynamic_caps::decode_dynamic_capability_ref_unverified(raw_ref) .map_err(|err| { protocol_error( @@ -1159,6 +1177,19 @@ pub(crate) fn inspect_dynamic_ref( "dynamic capability ref is bound to a different holder", )); } + let identity = dynamic_ref_issuer_identity(state)?; + if parsed.claims.issuer_id != identity.id { + return Err(protocol_error( + ProtocolErrorCode::MalformedRef, + "dynamic capability ref issuer is not this site controller", + )); + } + let signing_key = identity.signing_key().map_err(|err| { + protocol_error( + ProtocolErrorCode::ControlStateUnavailable, + &format!("site controller mesh identity is invalid: {err}"), + ) + })?; amber_mesh::dynamic_caps::verify_dynamic_capability_ref(&parsed, &signing_key.verifying_key()) .map_err(|err| { protocol_error( diff --git a/runtime/site-controller/src/lib.rs b/runtime/site-controller/src/lib.rs index 5c001d60..dfd6e4f3 100644 --- a/runtime/site-controller/src/lib.rs +++ b/runtime/site-controller/src/lib.rs @@ -80,12 +80,13 @@ pub use self::{ default_runtime::{ SiteControllerPeerRouterRoute, assign_compose_egress_network_subnets, cleanup_dynamic_site_children, host_service_bind_addr_for_consumer, - inject_compose_site_controller, inject_kubernetes_site_controller, - inject_site_controller_peer_router_routes, observability_endpoint_for_site, - prepare_kubernetes_artifact_namespace, reserve_host_port, reserve_loopback_port, - router_mesh_addr_for_consumer, set_compose_router_published_mesh_port, - set_site_artifact_mesh_identity_seed, site_controller_peer_router_url, - vm_endpoint_forward_ready_timeout, vm_uses_tcg_accel, walk_files, + inject_compose_site_controller, inject_compose_site_controller_with_mount_sources, + inject_kubernetes_site_controller, inject_site_controller_peer_router_routes, + observability_endpoint_for_site, prepare_kubernetes_artifact_namespace, reserve_host_port, + reserve_loopback_port, router_mesh_addr_for_consumer, + set_compose_router_published_mesh_port, set_site_artifact_mesh_identity_seed, + site_controller_peer_router_url, vm_endpoint_forward_ready_timeout, vm_uses_tcg_accel, + walk_files, }, runtime_api::{ DesiredExportPeerOverlay, DesiredExternalSlotOverlay, DesiredRouteOverlay, LaunchedSite, @@ -97,10 +98,10 @@ pub use self::{ }, state::{ DynamicInputDirectRecord, DynamicInputRouteTarget, DynamicProxyExportRecord, - SITE_CONTROLLER_INTERNAL_CAPABILITY, SITE_CONTROLLER_SERVICE_NAME, SiteControllerPlan, - authority_url_for_listen_addr, build_site_controller_state, - generate_control_state_auth_token, site_controller_internal_route_id, write_control_state, - write_site_controller_plan, + SITE_CONTROLLER_INTERNAL_CAPABILITY, SITE_CONTROLLER_MESH_IDENTITY_PATH, + SITE_CONTROLLER_SERVICE_NAME, SiteControllerPlan, authority_url_for_listen_addr, + build_site_controller_state, generate_control_state_auth_token, + site_controller_internal_route_id, write_control_state, write_site_controller_plan, }, }; diff --git a/runtime/site-controller/src/site_controller.rs b/runtime/site-controller/src/site_controller.rs index b0ed683b..374a8348 100644 --- a/runtime/site-controller/src/site_controller.rs +++ b/runtime/site-controller/src/site_controller.rs @@ -24,7 +24,7 @@ use super::{ self, ControlDynamicGrantAuthoritySyncRequest, ControlDynamicGrantAuthoritySyncResponse, ControlDynamicHeldDetailRequest, ControlDynamicHeldListRequest, ControlDynamicInspectRefRequest, ControlDynamicResolveOriginRequest, - ControlDynamicRevokeRequest, ControlDynamicShareRequest, + ControlDynamicResolveRefRequest, ControlDynamicRevokeRequest, ControlDynamicShareRequest, InternalDynamicResolveOriginRequest, }, http::{ @@ -61,6 +61,12 @@ pub(crate) async fn run_site_controller( let plan: SiteControllerPlan = read_json(plan_path.as_path(), "site controller plan")?; let mut control_state: FrameworkControlState = read_json(Path::new(&plan.state_path), "site controller state file")?; + if let Some(identity_path) = plan.controller_identity_path.as_deref() { + control_state.controller_identity = Some(read_json( + Path::new(identity_path), + "site controller mesh identity", + )?); + } persist_control_state(Path::new(&plan.state_path), &mut control_state)?; let control = ControlStateApp { control_state: Arc::new(Mutex::new(control_state)), @@ -216,6 +222,10 @@ pub(crate) fn site_controller_router(app_state: SiteControllerApp) -> Router { "/v1/controller/dynamic-caps/inspect-ref", post(control_dynamic_inspect_ref_route), ) + .route( + "/v1/controller/dynamic-caps/resolve-ref", + post(control_dynamic_resolve_ref_route), + ) .route( "/v1/controller/dynamic-caps/revoke", post(control_dynamic_revoke_route), @@ -614,6 +624,53 @@ async fn local_resolve_origin( } } +pub(super) async fn execute_site_controller_dynamic_caps_resolve_ref( + app: &SiteControllerApp, + request: ControlDynamicResolveRefRequest, +) -> std::result::Result { + let inspected = match execute_site_controller_dynamic_caps_inspect( + app, + DynamicCapsInspectRequest::InspectRef(ControlDynamicInspectRefRequest { + holder_component_id: request.holder_component_id.clone(), + r#ref: request.r#ref.clone(), + }), + false, + ) + .await? + { + DynamicCapsInspectResponse::InspectRef(response) => response, + _ => unreachable!("inspect_ref should return inspect response"), + }; + let origin = match execute_site_controller_dynamic_caps_inspect( + app, + DynamicCapsInspectRequest::ResolveOrigin(ControlDynamicResolveOriginRequest { + holder_component_id: request.holder_component_id, + source: dynamic_caps::DynamicCapabilityControlSourceRequest::Grant { + grant_id: inspected.grant_id.clone(), + }, + }), + false, + ) + .await? + { + DynamicCapsInspectResponse::ResolveOrigin(response) => response, + _ => unreachable!("resolve_origin should return origin resolution"), + }; + let parsed = amber_mesh::dynamic_caps::decode_dynamic_capability_ref_unverified(&request.r#ref) + .map_err(|err| { + ProtocolApiError::from(protocol_error( + ProtocolErrorCode::MalformedRef, + &format!("dynamic capability ref is malformed: {err}"), + )) + })?; + Ok(dynamic_caps::ControlDynamicResolveRefResponse { + origin, + relative_path: parsed.relative_path, + query: parsed.query, + fragment: parsed.fragment, + }) +} + async fn local_share( app: &SiteControllerApp, request: ControlDynamicShareRequest, @@ -1416,6 +1473,23 @@ async fn control_dynamic_inspect_ref_route( } } +async fn control_dynamic_resolve_ref_route( + State(app): State, + headers: HeaderMap, + Json(request): Json, +) -> std::result::Result, ProtocolApiError> { + let auth = authorize_dynamic_caps_request(&app, &headers, &request.holder_component_id).await?; + if auth != DynamicCapsRequestAuth::Sidecar { + return Err(ProtocolApiError::unauthorized( + "dynamic capability ref resolution must be requested by the holder component sidecar" + .to_string(), + )); + } + Ok(Json( + execute_site_controller_dynamic_caps_resolve_ref(&app, request).await?, + )) +} + async fn control_dynamic_revoke_route( State(app): State, headers: HeaderMap, diff --git a/runtime/site-controller/src/state.rs b/runtime/site-controller/src/state.rs index f6cdbea1..aee1437a 100644 --- a/runtime/site-controller/src/state.rs +++ b/runtime/site-controller/src/state.rs @@ -12,6 +12,7 @@ pub(super) const CONTROL_STATE_AUTH_HEADER: &str = "x-amber-control-state-auth"; pub const SITE_CONTROLLER_INTERNAL_CAPABILITY: &str = amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME; pub const SITE_CONTROLLER_SERVICE_NAME: &str = "amber-site-controller"; +pub const SITE_CONTROLLER_MESH_IDENTITY_PATH: &str = "/amber/mesh/mesh-identity.json"; pub fn site_controller_internal_route_id(site_id: &str) -> String { format!("site-controller:{site_id}") @@ -221,8 +222,8 @@ pub struct FrameworkControlState { pub(crate) capability_instances: BTreeMap, #[serde(default)] pub(crate) journal: Vec, - #[serde(default)] - pub(crate) dynamic_capability_signing_seed_b64: String, + #[serde(skip)] + pub(crate) controller_identity: Option, #[serde(default)] pub(crate) next_dynamic_capability_grant_id: u64, #[serde(default)] @@ -274,7 +275,8 @@ pub struct SiteControllerPlan { pub site_state_root: String, pub artifact_dir: String, pub control_state_auth_token: String, - pub dynamic_caps_token_verify_key_b64: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub controller_identity_path: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub storage_root: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -300,15 +302,18 @@ pub(crate) fn build_control_state( run_id: &str, run_plan: &RunPlan, ) -> Result { - build_control_state_with_signing_seed( + let mut state = build_control_state_base(run_id, run_plan)?; + state.controller_identity = Some(test_controller_identity(run_id)); + Ok(state) +} + +#[cfg(test)] +pub(crate) fn test_controller_identity(run_id: &str) -> amber_mesh::MeshIdentitySecret { + amber_mesh::MeshIdentitySecret::from_identity(&amber_mesh::MeshIdentity::derive( + "/site/test/controller", + Some("test".to_string()), run_id, - run_plan, - &amber_mesh::dynamic_caps::signing_seed_b64( - &amber_mesh::dynamic_caps::signing_key_from_seed( - amber_mesh::dynamic_caps::generate_dynamic_capability_signing_seed(), - ), - ), - ) + )) } pub fn build_site_controller_state( @@ -317,13 +322,8 @@ pub fn build_site_controller_state( site_id: &str, site_index: usize, site_count: usize, - dynamic_capability_signing_seed_b64: &str, ) -> Result { - let mut state = build_control_state_with_signing_seed( - run_id, - run_plan, - dynamic_capability_signing_seed_b64, - )?; + let mut state = build_control_state_base(run_id, run_plan)?; localize_framework_control_state(&mut state, site_id)?; let site_offset = site_index as u64; let id_stride = site_count.max(1) as u64; @@ -410,11 +410,7 @@ fn dynamic_capability_grant_counter(grant_id: &str) -> Option { .and_then(|suffix| u64::from_str_radix(suffix, 16).ok()) } -fn build_control_state_with_signing_seed( - run_id: &str, - run_plan: &RunPlan, - dynamic_capability_signing_seed_b64: &str, -) -> Result { +fn build_control_state_base(run_id: &str, run_plan: &RunPlan) -> Result { let scenario = Scenario::try_from(run_plan.base_scenario.clone()) .into_diagnostic() .map_err(|err| miette::miette!("failed to decode frozen base scenario: {err}"))?; @@ -448,7 +444,7 @@ fn build_control_state_with_signing_seed( next_component_id, capability_instances: BTreeMap::new(), journal: Vec::new(), - dynamic_capability_signing_seed_b64: dynamic_capability_signing_seed_b64.to_string(), + controller_identity: None, next_dynamic_capability_grant_id: 0, dynamic_capability_grants: BTreeMap::new(), dynamic_capability_grant_authority_sites: BTreeMap::new(), @@ -1243,7 +1239,7 @@ pub fn write_site_controller_plan( site_state_root: &Path, artifact_dir: &Path, control_state_auth_token: &str, - dynamic_caps_token_verify_key_b64: &str, + controller_identity_path: Option<&str>, storage_root: Option<&str>, runtime_root: Option<&str>, router_mesh_port: Option, @@ -1277,7 +1273,7 @@ pub fn write_site_controller_plan( site_state_root: site_state_root.display().to_string(), artifact_dir: artifact_dir.display().to_string(), control_state_auth_token: control_state_auth_token.to_string(), - dynamic_caps_token_verify_key_b64: dynamic_caps_token_verify_key_b64.to_string(), + controller_identity_path: controller_identity_path.map(str::to_string), storage_root: storage_root.map(str::to_string), runtime_root: runtime_root.map(str::to_string), router_mesh_port, diff --git a/runtime/site-controller/src/tests.rs b/runtime/site-controller/src/tests.rs index c944e105..a784726f 100644 --- a/runtime/site-controller/src/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -2747,7 +2747,7 @@ fn test_control_state_app( site_state_root: site_state_root.display().to_string(), artifact_dir: artifact_dir.display().to_string(), control_state_auth_token: "test-control-state-auth".to_string(), - dynamic_caps_token_verify_key_b64: String::new(), + controller_identity_path: None, storage_root: Some(storage_root.display().to_string()), runtime_root: Some(runtime_root.display().to_string()), router_mesh_port: Some(24000), @@ -3105,6 +3105,84 @@ fn install_remote_controller_peer_url_fixture( plan.peer_site_router_urls.extend(remote_base_urls.clone()); } +fn retarget_test_controller_site(app: &mut ControlStateApp, site_id: &str) { + let plan = Arc::make_mut(&mut app.controller_plan); + plan.site_id = site_id.to_string(); + plan.router_identity_id = format!("/site/{site_id}/router"); + plan.site_state_root = app.state_root.join(site_id).display().to_string(); + if let Some(parent) = Path::new(&plan.artifact_dir).parent() { + plan.artifact_dir = parent.join(site_id).display().to_string(); + } + if let Some(parent) = plan + .runtime_root + .as_deref() + .and_then(|runtime_root| Path::new(runtime_root).parent()) + { + plan.runtime_root = Some(parent.join(site_id).display().to_string()); + } + if let Some(parent) = plan + .storage_root + .as_deref() + .and_then(|storage_root| Path::new(storage_root).parent()) + { + plan.storage_root = Some(parent.join(site_id).display().to_string()); + } +} + +fn install_bob_holder_runtime_fixture(app: &ControlStateApp) { + let plan = app.controller_plan.as_ref(); + let artifact_dir = PathBuf::from(&plan.artifact_dir); + let runtime_root = PathBuf::from( + plan.runtime_root + .as_deref() + .expect("test controller plan should include runtime root"), + ); + fs::create_dir_all(&artifact_dir).expect("artifact dir should exist"); + fs::create_dir_all(&runtime_root).expect("runtime root should exist"); + write_json( + &artifact_dir.join("direct-plan.json"), + &json!({ + "components": [{ + "id": 3, + "moniker": "/bob", + "sidecar": { + "mesh_config_path": "bob-mesh.json", + }, + }], + "router": { + "mesh_config_path": "router-mesh.json", + }, + }), + ) + .expect("holder direct plan should write"); + write_json( + &direct_runtime_state_path(&artifact_dir), + &DirectRuntimeState { + component_mesh_port_by_id: BTreeMap::from([(3, 24003)]), + ..Default::default() + }, + ) + .expect("holder direct runtime state should write"); + write_json( + &runtime_root.join("bob-mesh.json"), + &test_live_component_runtime( + "/bob", + "/bob", + "127.0.0.1:24003", + "127.0.0.1:24003", + Vec::new(), + Vec::new(), + ) + .mesh_config, + ) + .expect("holder mesh config should write"); + write_json( + &runtime_root.join("router-mesh.json"), + &test_live_site_router(Vec::new()), + ) + .expect("holder router mesh config should write"); +} + struct TestMcpClient { client: Client, endpoint: String, @@ -4391,9 +4469,7 @@ async fn inspect_ref_routes_remote_grants_via_synced_authority_site() { let (authority_base_url, _authority_handle) = spawn_test_router(router).await; let mut app = test_control_state_app(&dir, holder_state, state_path); - let controller_plan = Arc::make_mut(&mut app.controller_plan); - controller_plan.site_id = "direct_b".to_string(); - controller_plan.router_identity_id = "/site/direct_b/router".to_string(); + retarget_test_controller_site(&mut app, "direct_b"); install_remote_controller_peer_url_fixture( &mut app, &BTreeMap::from([("direct_a".to_string(), authority_base_url)]), @@ -4431,6 +4507,196 @@ async fn inspect_ref_routes_remote_grants_via_synced_authority_site() { ); } +#[tokio::test] +async fn resolve_ref_routes_remote_grants_and_preserves_ref_suffix() { + let dir = TempDir::new().expect("temp dir"); + let placement = PlacementFile { + schema: amber_compiler::run_plan::PLACEMENT_SCHEMA.to_string(), + version: amber_compiler::run_plan::PLACEMENT_VERSION, + sites: BTreeMap::from([ + ( + "direct_a".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ( + "direct_b".to_string(), + SiteDefinition { + kind: SiteKind::Direct, + context: None, + controller_site: None, + }, + ), + ]), + defaults: PlacementDefaults { + path: Some("direct_a".to_string()), + ..PlacementDefaults::default() + }, + components: BTreeMap::from([ + ("/provider".to_string(), "direct_a".to_string()), + ("/alice".to_string(), "direct_a".to_string()), + ("/bob".to_string(), "direct_b".to_string()), + ]), + dynamic_capabilities: None, + framework_children: None, + }; + let mut authoritative = compile_control_state_from_ir_with_run_id( + dynamic_caps_binding_scenario_ir(), + Some(&placement), + "test-run", + ) + .await; + let alice_root = super::dynamic_caps::source_key_from_held_id( + &authoritative, + "components./alice", + &root_held_id_for(&authoritative, "components./alice"), + ) + .expect("alice root source should resolve"); + let share = super::dynamic_caps::share_dynamic_capability( + &mut authoritative, + "components./alice", + &alice_root, + "components./bob", + None, + &json!({}), + ) + .expect("cross-site share should succeed"); + let (grant_id, shared_ref) = match share { + super::dynamic_caps::DynamicCapabilityShareOutcome::Created { grant_id, r#ref } => { + (grant_id, r#ref) + } + _ => panic!("cross-site share should create a grant"), + }; + let mut suffixed_ref = Url::parse(&shared_ref).expect("shared ref should be a URL"); + let token_path = suffixed_ref.path().trim_end_matches('/'); + suffixed_ref.set_path(&format!("{token_path}/nested/path")); + suffixed_ref.set_query(Some("q=1")); + suffixed_ref.set_fragment(Some("section")); + let suffixed_ref = suffixed_ref.to_string(); + + let mut holder_state = authoritative.clone(); + localize_framework_control_state(&mut holder_state, "direct_b") + .expect("holder site state should localize"); + let state_path = dir.path().join("control-state.json"); + write_control_state(&state_path, &holder_state).expect("holder state should write"); + + let hits = Arc::new(std::sync::Mutex::new(Vec::::new())); + let remote_grant_id = grant_id.clone(); + let router = Router::new() + .route( + "/v1/controller/dynamic-caps/inspect-ref", + axum::routing::post({ + let hits = hits.clone(); + let remote_grant_id = remote_grant_id.clone(); + move |_headers: HeaderMap, + Json(request): Json| { + let hits = hits.clone(); + let remote_grant_id = remote_grant_id.clone(); + async move { + hits.lock() + .expect("resolve-ref hit log poisoned") + .push(format!("inspect:{}", request.r#ref)); + Json(amber_mesh::dynamic_caps::InspectRefResponse { + state: HeldEntryState::Live, + grant_id: remote_grant_id.clone(), + holder_component_id: "components./bob".to_string(), + descriptor: DescriptorIr { + kind: "http".to_string(), + label: "provider.http".to_string(), + profile: None, + }, + held_id: Some(super::dynamic_caps::held_id_for_grant( + &remote_grant_id, + )), + }) + } + } + }), + ) + .route( + "/v1/internal/dynamic-caps/resolve-origin", + axum::routing::post({ + let hits = hits.clone(); + let remote_grant_id = remote_grant_id.clone(); + move |_headers: HeaderMap, + Json(request): Json| { + let hits = hits.clone(); + let remote_grant_id = remote_grant_id.clone(); + async move { + let dynamic_caps::DynamicCapabilityControlSourceRequest::Grant { + grant_id, + } = request.source + else { + panic!("resolve-ref should resolve the inspected grant"); + }; + hits.lock() + .expect("resolve-ref hit log poisoned") + .push(format!("resolve:{grant_id}")); + Json(dynamic_caps::ControlDynamicResolveOriginResponse { + held_id: super::dynamic_caps::held_id_for_grant(&remote_grant_id), + descriptor: DescriptorIr { + kind: "http".to_string(), + label: "provider.http".to_string(), + profile: None, + }, + origin_route_id: "dynamic-origin".to_string(), + origin_capability: "provider.http".to_string(), + origin_protocol: "http".to_string(), + origin_peer_id: "/site/direct_a/router".to_string(), + origin_peer_key_b64: base64::engine::general_purpose::STANDARD + .encode([9u8; 32]), + origin_peer_addr: "127.0.0.1:24000".to_string(), + }) + } + } + }), + ); + let (authority_base_url, _authority_handle) = spawn_test_router(router).await; + + let mut app = test_control_state_app(&dir, holder_state, state_path); + retarget_test_controller_site(&mut app, "direct_b"); + install_remote_controller_peer_url_fixture( + &mut app, + &BTreeMap::from([("direct_a".to_string(), authority_base_url)]), + ); + install_bob_holder_runtime_fixture(&app); + let controller_app = SiteControllerApp { + control: app, + ready: ready_site_controller_flag(), + }; + + let response = super::site_controller::execute_site_controller_dynamic_caps_resolve_ref( + &controller_app, + dynamic_caps::ControlDynamicResolveRefRequest { + holder_component_id: "components./bob".to_string(), + r#ref: suffixed_ref.clone(), + }, + ) + .await + .expect("holder site should resolve remote refs through the authority site"); + + assert_eq!( + response.origin.held_id, + super::dynamic_caps::held_id_for_grant(&grant_id) + ); + assert_eq!(response.relative_path, "/nested/path"); + assert_eq!(response.query.as_deref(), Some("q=1")); + assert_eq!(response.fragment.as_deref(), Some("section")); + assert_eq!( + hits.lock() + .expect("resolve-ref hit log poisoned") + .as_slice(), + &[ + format!("inspect:{suffixed_ref}"), + format!("resolve:{grant_id}") + ], + "resolve_ref should validate the raw ref and then resolve the inspected grant", + ); +} + #[tokio::test] async fn held_list_aggregates_remote_grants_via_remote_controller_capability() { let dir = TempDir::new().expect("temp dir"); @@ -5364,17 +5630,9 @@ async fn open_template_replay_uses_admitted_manifest_after_source_mutation() { CompiledScenario::from_ir(replay_scenario_ir).expect("snapshot replay should compile"); let replay_run_plan = build_run_plan(&replay_compiled, Some(&replay_placement)).expect("replay run plan"); - let mut localized_replay = build_site_controller_state( - "replay-run", - &replay_run_plan, - "direct_local", - 0, - 1, - &mesh_dynamic_caps::signing_seed_b64(&mesh_dynamic_caps::signing_key_from_seed( - mesh_dynamic_caps::generate_dynamic_capability_signing_seed(), - )), - ) - .expect("site-local replay state should build"); + let mut localized_replay = + build_site_controller_state("replay-run", &replay_run_plan, "direct_local", 0, 1) + .expect("site-local replay state should build"); assert!( localized_replay .live_children @@ -7350,11 +7608,7 @@ fn shared_cross_site_link_is_retained_while_another_child_still_needs_it() { next_tx_id: 0, id_stride: 1, next_component_id: 0, - dynamic_capability_signing_seed_b64: mesh_dynamic_caps::signing_seed_b64( - &mesh_dynamic_caps::signing_key_from_seed( - mesh_dynamic_caps::generate_dynamic_capability_signing_seed(), - ), - ), + controller_identity: None, next_dynamic_capability_grant_id: 0, dynamic_capability_grants: BTreeMap::new(), dynamic_capability_grant_authority_sites: BTreeMap::new(), @@ -9770,13 +10024,18 @@ async fn dynamic_capabilities_inspect_ref_rejects_unsupported_token_versions() { super::dynamic_caps::DynamicCapabilityShareOutcome::Created { grant_id, .. } => grant_id, other => panic!("unexpected share outcome: {other:?}"), }; - let signing_key = - mesh_dynamic_caps::signing_key_from_seed_b64(&state.dynamic_capability_signing_seed_b64) - .expect("test signing key should decode"); + let identity = state + .controller_identity + .as_ref() + .expect("test controller identity should be loaded"); + let signing_key = identity + .signing_key() + .expect("test controller identity should be valid"); let unsupported_ref = mesh_dynamic_caps::build_dynamic_capability_ref_url( DynamicCapabilityRefClaims { version: mesh_dynamic_caps::DYNAMIC_CAPS_REF_VERSION + 1, run_id: state.run_id.clone(), + issuer_id: identity.id.clone(), grant_id, holder_component_id: "components./carol".to_string(), descriptor_hint: Some("provider.http".to_string()), @@ -9852,7 +10111,7 @@ volumes: &site_state_root, &artifact_root, "test-auth", - "test-verify-key", + Some(SITE_CONTROLLER_MESH_IDENTITY_PATH), None, None, Some(24000), @@ -10120,7 +10379,7 @@ spec: &site_state_root, &artifact_root, "test-auth", - "test-verify-key", + Some(SITE_CONTROLLER_MESH_IDENTITY_PATH), None, None, Some(24000), @@ -10276,11 +10535,7 @@ fn local_site_manager_state_uses_controller_plan_when_host_state_is_absent() { next_component_id: 0, capability_instances: BTreeMap::new(), journal: Vec::new(), - dynamic_capability_signing_seed_b64: mesh_dynamic_caps::signing_seed_b64( - &mesh_dynamic_caps::signing_key_from_seed( - mesh_dynamic_caps::generate_dynamic_capability_signing_seed(), - ), - ), + controller_identity: None, next_dynamic_capability_grant_id: 0, dynamic_capability_grants: BTreeMap::new(), dynamic_capability_grant_authority_sites: BTreeMap::new(), @@ -10368,11 +10623,7 @@ fn site_execution_site_routes_vm_sites_to_their_direct_controller() { next_component_id: 0, capability_instances: BTreeMap::new(), journal: Vec::new(), - dynamic_capability_signing_seed_b64: mesh_dynamic_caps::signing_seed_b64( - &mesh_dynamic_caps::signing_key_from_seed( - mesh_dynamic_caps::generate_dynamic_capability_signing_seed(), - ), - ), + controller_identity: None, next_dynamic_capability_grant_id: 0, dynamic_capability_grants: BTreeMap::new(), dynamic_capability_grant_authority_sites: BTreeMap::new(), @@ -10437,11 +10688,7 @@ fn load_site_manager_state_prefers_local_controller_view_over_stale_host_state() next_component_id: 0, capability_instances: BTreeMap::new(), journal: Vec::new(), - dynamic_capability_signing_seed_b64: mesh_dynamic_caps::signing_seed_b64( - &mesh_dynamic_caps::signing_key_from_seed( - mesh_dynamic_caps::generate_dynamic_capability_signing_seed(), - ), - ), + controller_identity: None, next_dynamic_capability_grant_id: 0, dynamic_capability_grants: BTreeMap::new(), dynamic_capability_grant_authority_sites: BTreeMap::new(), From 416929e829e001e3b22185d20b27b472f531a2f3 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Sat, 9 May 2026 21:34:58 +0800 Subject: [PATCH 22/25] Fix docker gateway permissions --- Cargo.lock | 2 +- Cargo.toml | 1 + cli/src/command_support.rs | 27 ++ cli/src/tests.rs | 44 +++ cli/tests/mixed_run/support.rs | 11 +- compiler/manifest/README.md | 17 +- compiler/manifest/src/document.rs | 5 +- compiler/manifest/src/error.rs | 8 + compiler/manifest/src/manifest/mod.rs | 33 ++ .../manifest/src/manifest/tests/components.rs | 67 +++- compiler/scenario/README.md | 5 + compiler/src/targets/direct/mod.rs | 3 +- .../framework_docker_injection.rs | 9 +- .../src/targets/mesh/docker_compose/mod.rs | 122 ++----- .../targets/mesh/docker_compose/tests/mod.rs | 13 +- .../mesh/docker_compose/tests/rendering.rs | 168 +++++++-- compiler/src/targets/mesh/internal_images.rs | 7 +- .../src/targets/mesh/kubernetes/tests/mod.rs | 1 + .../mesh/kubernetes/tests/rendering.rs | 108 ++++++ compiler/src/targets/mesh/mesh_config.rs | 37 +- compiler/src/targets/vm/mod.rs | 3 +- docker/amber-cli/Dockerfile | 3 +- docker/amber-docker-gateway/Dockerfile | 75 ---- docker/amber-helper/Dockerfile | 2 +- docker/amber-manager/Dockerfile | 2 +- docker/amber-provisioner/Dockerfile | 2 +- docker/amber-router/Dockerfile | 3 +- docker/amber-site-controller/Dockerfile | 3 +- docker/images.json | 6 - images/src/lib.rs | 9 +- runtime/docker-gateway/Cargo.toml | 3 +- runtime/docker-gateway/README.md | 94 ++--- runtime/docker-gateway/src/lib.rs | 326 +++--------------- runtime/docker-gateway/src/main.rs | 29 -- runtime/docker-gateway/src/tests/compose.rs | 16 +- runtime/docker-gateway/src/tests/e2e.rs | 99 ++---- runtime/docker-gateway/src/tests/gateway.rs | 190 +++------- runtime/docker-gateway/src/tests/mod.rs | 114 +++--- runtime/docker-gateway/src/tests/unit.rs | 36 +- runtime/helper/src/lib.rs | 77 +++++ runtime/helper/src/main.rs | 19 +- runtime/mesh/src/lib.rs | 4 + runtime/router/Cargo.toml | 1 + runtime/router/src/control.rs | 63 ++++ runtime/router/src/lib.rs | 25 ++ runtime/router/src/tests.rs | 29 ++ runtime/router/src/transport.rs | 18 + 47 files changed, 965 insertions(+), 974 deletions(-) delete mode 100644 docker/amber-docker-gateway/Dockerfile delete mode 100644 runtime/docker-gateway/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 2f80f6cc..b9e6eac4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -184,7 +184,6 @@ dependencies = [ name = "amber-docker-gateway" version = "0.0.0" dependencies = [ - "base64", "bytes", "dashmap", "http-body-util", @@ -377,6 +376,7 @@ name = "amber-router" version = "0.0.0" dependencies = [ "aho-corasick", + "amber-docker-gateway", "amber-mesh", "base64", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 44b23348..2501f921 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ resolver = "2" [workspace.dependencies] amber-compiler = { path = "compiler" } amber-config = { path = "compiler/config" } +amber-docker-gateway = { path = "runtime/docker-gateway" } amber-images = { path = "images" } amber-json5 = { path = "compiler/json5" } amber-manifest = { path = "compiler/manifest" } diff --git a/cli/src/command_support.rs b/cli/src/command_support.rs index 25861d55..45630405 100644 --- a/cli/src/command_support.rs +++ b/cli/src/command_support.rs @@ -295,12 +295,39 @@ pub(crate) fn load_compiled_scenario_ir(path: &Path) -> Result Result<()> { + for component in &ir.components { + for (provide_name, provide) in &component.provides { + let kind = provide.decl.kind; + if matches!( + kind, + amber_manifest::CapabilityKind::Component + | amber_manifest::CapabilityKind::Docker + | amber_manifest::CapabilityKind::Kvm + ) { + return Err(miette::miette!( + "Scenario IR input `{}` component `{}` provide `{}` declares framework-owned \ + capability kind `{}`; use `framework.{}` as the source instead of declaring \ + a provider", + path.display(), + component.moniker, + provide_name, + kind, + kind.as_str() + )); + } + } + } + Ok(()) +} + pub(crate) fn is_run_plan_file(path: &Path) -> Result { if !path.is_file() { return Ok(false); diff --git a/cli/src/tests.rs b/cli/src/tests.rs index 68a72867..c83d8086 100644 --- a/cli/src/tests.rs +++ b/cli/src/tests.rs @@ -101,6 +101,50 @@ fn top_level_help_groups_authoring_and_runtime_workflows() { ); } +#[test] +fn scenario_ir_input_rejects_framework_owned_provide_kinds() { + for kind in ["component", "docker", "kvm"] { + let temp = tempfile::tempdir().expect("tempdir should exist"); + let path = temp.path().join(format!("{kind}.json")); + let ir = serde_json::json!({ + "schema": SCENARIO_IR_SCHEMA, + "version": amber_scenario::SCENARIO_IR_VERSION, + "root": 0, + "components": [ + { + "id": 0, + "moniker": "/", + "parent": null, + "children": [], + "digest": amber_manifest::ManifestDigest::new([0u8; 32]).to_string(), + "config": null, + "program": null, + "slots": {}, + "provides": { + "api": { + "kind": kind, + "endpoint": "api" + } + } + } + ], + "bindings": [], + "exports": [] + }); + fs::write( + &path, + serde_json::to_vec_pretty(&ir).expect("IR should serialize"), + ) + .expect("IR should be written"); + + let err = load_compiled_scenario_ir(&path) + .expect_err("framework-owned provide kind should be rejected"); + let rendered = err.to_string(); + assert!(rendered.contains("framework-owned"), "{rendered}"); + assert!(rendered.contains(kind), "{rendered}"); + } +} + #[test] fn attached_run_overview_stays_focused_on_identity_and_exports() { let mut output = Vec::new(); diff --git a/cli/tests/mixed_run/support.rs b/cli/tests/mixed_run/support.rs index 5bba3fcf..701e9f64 100644 --- a/cli/tests/mixed_run/support.rs +++ b/cli/tests/mixed_run/support.rs @@ -30,9 +30,8 @@ use std::{ }; use amber_images::{ - AMBER_DOCKER_GATEWAY, AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER, AMBER_SITE_CONTROLLER, - DEV_IMAGE_TAGS_ENV, INTERNAL_IMAGE_OVERRIDE_KEYS, ImageRef, override_reference, - parse_dev_image_tag_overrides, + AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER, AMBER_SITE_CONTROLLER, DEV_IMAGE_TAGS_ENV, + INTERNAL_IMAGE_OVERRIDE_KEYS, ImageRef, override_reference, parse_dev_image_tag_overrides, }; use cloud_image_support::default_host_arch_cloud_image_filename; use outputs_root_support::cli_test_outputs_root; @@ -1251,10 +1250,6 @@ pub(crate) fn ensure_amber_internal_images() { &images.site_controller, &root.join("docker/amber-site-controller/Dockerfile"), ); - ensure_docker_image( - &images.docker_gateway, - &root.join("docker/amber-docker-gateway/Dockerfile"), - ); }); } @@ -1271,7 +1266,6 @@ struct AmberInternalImageRefs { router: String, helper: String, provisioner: String, - docker_gateway: String, site_controller: String, } @@ -1288,7 +1282,6 @@ fn amber_internal_image_refs() -> AmberInternalImageRefs { router: resolve(&AMBER_ROUTER, "router"), helper: resolve(&AMBER_HELPER, "helper"), provisioner: resolve(&AMBER_PROVISIONER, "provisioner"), - docker_gateway: resolve(&AMBER_DOCKER_GATEWAY, "docker_gateway"), site_controller: resolve(&AMBER_SITE_CONTROLLER, "site_controller"), } } diff --git a/compiler/manifest/README.md b/compiler/manifest/README.md index e186ed91..cd24ce64 100644 --- a/compiler/manifest/README.md +++ b/compiler/manifest/README.md @@ -132,6 +132,7 @@ Top-level object: Current values: * `"docker"` +* `"kvm"` Rules: @@ -344,7 +345,7 @@ Supported `from` sources (current): * `slots.`: mount a storage slot as a directory. The referenced slot must exist and have `kind: "storage"`. * `framework.docker`: requires `experimental_features: ["docker"]`. In Docker Compose output, this - injects a Docker socket mount backed by the framework docker gateway. + injects a Docker socket mount backed by the router-owned framework docker gateway. * `framework.kvm`: requires `experimental_features: ["kvm"]`. In Docker Compose output, adds a `/dev/kvm` device mapping to the container for hardware-accelerated virtualization (QEMU/KVM). Also adds the `kvm` group to the container via `group_add`, using the `AMBER_KVM_GID` environment @@ -352,9 +353,11 @@ Supported `from` sources (current): `stat -c %g /dev/kvm`). The generated `env.example` includes this variable. Other targets (e.g. Kubernetes) do not yet support this capability. -Reserved (not implemented yet): +Reserved (not implemented for mounts): -* `framework.` other than `framework.docker` and `framework.kvm` +* `framework.component` exists only as a binding source; it cannot be mounted. +* `framework.` other than `framework.component`, `framework.docker`, and + `framework.kvm` Mount value formatting: @@ -537,9 +540,9 @@ config_schema: { ### Capability declaration shape -Both slots and provides share: +Slots, resources, and provides use capability declarations: -* `kind`: `"mcp" | "llm" | "http" | "docker" | "a2a" | "storage"` +* `kind`: `"mcp" | "llm" | "http" | "component" | "docker" | "a2a" | "storage" | "kvm"` * `profile` (optional): string qualifier (often used for `"mcp"`) Example: @@ -659,6 +662,9 @@ Notes: * This crate enforces that each provide declares an `endpoint` and that it refers to a declared endpoint name. * `provides` cannot declare `kind: "storage"`. Storage is routed through `slots` and `bindings`, then consumed via `program.mounts`. +* `provides` cannot declare framework-owned kinds: `"component"`, `"docker"`, or `"kvm"`. + Those capability sources are created only by `framework.component`, `framework.docker`, and + `framework.kvm`. * To forward a child capability, use `exports` pointing at `#child.`. ### `exports` @@ -774,6 +780,7 @@ Framework capabilities are a fixed compiler-known list. Current framework capabilities: +* `component` (`framework.component`) — framework-provided dynamic child component management. * `docker` (`framework.docker`) — requires `experimental_features: ["docker"]` in the same manifest. * `kvm` (`framework.kvm`) — requires `experimental_features: ["kvm"]`. Docker Compose only. Adds a `/dev/kvm` device mapping to the container. diff --git a/compiler/manifest/src/document.rs b/compiler/manifest/src/document.rs index c60dfc55..6d2d0813 100644 --- a/compiler/manifest/src/document.rs +++ b/compiler/manifest/src/document.rs @@ -198,7 +198,10 @@ fn labels_for_manifest_error( ManifestError::MissingProvideEndpoint { name } => { labels_for_missing_provide_endpoint(spans, name) } - ManifestError::UnsupportedProvideKind { name, .. } => labels_for_provide_kind(spans, name), + ManifestError::UnsupportedProvideKind { name, .. } + | ManifestError::FrameworkOwnedProvideKind { name, .. } => { + labels_for_provide_kind(spans, name) + } ManifestError::UnsupportedResourceKind { name, .. } => { labels_for_resource_decl(spans, name, "unsupported resource kind here") } diff --git a/compiler/manifest/src/error.rs b/compiler/manifest/src/error.rs index 809ba87a..c1b3f3a6 100644 --- a/compiler/manifest/src/error.rs +++ b/compiler/manifest/src/error.rs @@ -211,6 +211,14 @@ pub enum Error { kind: crate::CapabilityKind, }, + #[error("provide `{name}` cannot declare framework-owned capability kind `{kind}`")] + #[diagnostic(code(manifest::framework_owned_provide_kind), help("{help}"))] + FrameworkOwnedProvideKind { + name: String, + kind: crate::CapabilityKind, + help: String, + }, + #[error("duplicate mount name `{name}`")] #[diagnostic(code(manifest::duplicate_mount_name))] DuplicateMountName { name: String }, diff --git a/compiler/manifest/src/manifest/mod.rs b/compiler/manifest/src/manifest/mod.rs index beedc171..9bd48d5f 100644 --- a/compiler/manifest/src/manifest/mod.rs +++ b/compiler/manifest/src/manifest/mod.rs @@ -248,6 +248,31 @@ fn validate_resource_decls(resources: &BTreeMap) -> Ok(()) } +fn framework_owned_provide_kind_help(kind: CapabilityKind) -> String { + match kind { + CapabilityKind::Kvm => "`kvm` access is created by `framework.kvm`. Use `program.mounts` \ + with `from: \"framework.kvm\"` instead of declaring a `provides` \ + entry." + .to_string(), + CapabilityKind::Component | CapabilityKind::Docker => { + let framework_capability = kind.as_str(); + format!( + "`{kind}` capabilities are created by `framework.{framework_capability}`. Declare \ + a `slots.: {{ kind: \"{kind}\" }}` consumer slot and bind it from \ + `framework.{framework_capability}` instead of declaring a `provides` entry." + ) + } + _ => "`provides` cannot declare this framework-owned capability kind.".to_string(), + } +} + +fn is_framework_owned_provide_kind(kind: CapabilityKind) -> bool { + matches!( + kind, + CapabilityKind::Component | CapabilityKind::Docker | CapabilityKind::Kvm + ) +} + fn validate_environment_extends( environments: &BTreeMap, ) -> Result<(), Error> { @@ -649,6 +674,14 @@ fn validate_endpoints( } for (provide_name, provide) in provides { + if is_framework_owned_provide_kind(provide.decl.kind) { + return Err(Error::FrameworkOwnedProvideKind { + name: provide_name.to_string(), + kind: provide.decl.kind, + help: framework_owned_provide_kind_help(provide.decl.kind), + }); + } + if provide.decl.kind == CapabilityKind::Storage { return Err(Error::UnsupportedProvideKind { name: provide_name.to_string(), diff --git a/compiler/manifest/src/manifest/tests/components.rs b/compiler/manifest/src/manifest/tests/components.rs index 3187dcd5..720a0b1e 100644 --- a/compiler/manifest/src/manifest/tests/components.rs +++ b/compiler/manifest/src/manifest/tests/components.rs @@ -396,31 +396,66 @@ fn endpoint_validation_still_rejects_unknown_provide_with_conditional_literal_en } #[test] -fn docker_capability_kind_parses_for_slots_and_provides() { +fn framework_owned_capability_kinds_parse_for_slots() { let m: Manifest = r#" { manifest_version: "0.1.0", - program: { - image: "x", - entrypoint: ["x"], - network: { endpoints: [ { name: "endpoint", port: 80 } ] } - }, slots: { - worker: { kind: "docker" } + component: { kind: "component" }, + docker: { kind: "docker" }, + kvm: { kind: "kvm", optional: true }, }, - provides: { - api: { kind: "docker", endpoint: "endpoint" } - }, - exports: { api: "api" }, } "# .parse() .unwrap(); - let worker = m.slots.get("worker").expect("worker slot"); - assert_eq!(worker.decl.kind, CapabilityKind::Docker); + let component = m.slots.get("component").expect("component slot"); + assert_eq!(component.decl.kind, CapabilityKind::Component); - let api = m.provides.get("api").expect("api provide"); - assert_eq!(api.decl.kind, CapabilityKind::Docker); - assert_eq!(api.endpoint.as_deref(), Some("endpoint")); + let docker = m.slots.get("docker").expect("docker slot"); + assert_eq!(docker.decl.kind, CapabilityKind::Docker); + + let kvm = m.slots.get("kvm").expect("kvm slot"); + assert_eq!(kvm.decl.kind, CapabilityKind::Kvm); +} + +#[test] +fn framework_owned_capability_kinds_cannot_be_provided_by_manifests() { + for (kind, expected) in [ + ("component", CapabilityKind::Component), + ("docker", CapabilityKind::Docker), + ("kvm", CapabilityKind::Kvm), + ] { + let raw = parse_raw(&format!( + r#" + {{ + manifest_version: "0.1.0", + program: {{ + image: "x", + entrypoint: ["x"], + network: {{ endpoints: [ {{ name: "endpoint", port: 80 }} ] }} + }}, + provides: {{ + api: {{ kind: "{kind}", endpoint: "endpoint" }} + }}, + exports: {{ api: "api" }}, + }} + "# + )); + + let err = match raw.validate() { + Ok(_) => panic!("{kind} provide should be rejected"), + Err(err) => err, + }; + match err { + Error::FrameworkOwnedProvideKind { + name, kind: actual, .. + } => { + assert_eq!(name, "api"); + assert_eq!(actual, expected); + } + other => panic!("expected FrameworkOwnedProvideKind for {kind}, got: {other}"), + } + } } diff --git a/compiler/scenario/README.md b/compiler/scenario/README.md index ba41b43e..979073e2 100644 --- a/compiler/scenario/README.md +++ b/compiler/scenario/README.md @@ -10,6 +10,11 @@ Data model for the fully linked scenario produced by the compiler. - `ScenarioIr`: serde-friendly JSON IR wrapper with schema/version headers and conversions to/from `Scenario`. - `ScenarioIrError`: validation errors when loading IR into a `Scenario`. +User-loaded Scenario IR follows the same framework capability ownership rules as manifests: +component, Docker, and KVM capability providers are framework-owned. User-authored IR should +represent those sources as framework bindings (`framework.component`, `framework.docker`, or +`framework.kvm`), not as component `provides` entries. + ## Graph utilities - `graph::topo_order`: dependency ordering by non-weak component bindings with cycle detection (framework and external bindings do not introduce component dependencies). - `graph::component_path[_for]`: stable path strings for diagnostics. diff --git a/compiler/src/targets/direct/mod.rs b/compiler/src/targets/direct/mod.rs index da7bde62..ee023e9d 100644 --- a/compiler/src/targets/direct/mod.rs +++ b/compiler/src/targets/direct/mod.rs @@ -32,7 +32,7 @@ use crate::{ }, mesh_config::{ MeshAddressing, MeshConfigBuildInput, MeshConfigBuildOptions, RouterPorts, - build_mesh_config_plan, + build_mesh_config_plan, default_mesh_config_build_options, }, plan::{MeshOptions, MeshPlan, build_mesh_plan, map_program_components}, ports::placeholder_local_route_ports, @@ -322,6 +322,7 @@ pub(crate) fn emit_direct_artifact_with_options( router_mesh_listen_addr: "127.0.0.1", router_control_listen_addr: "127.0.0.1", force_router: options.force_router, + ..default_mesh_config_build_options() }, })?; diff --git a/compiler/src/targets/mesh/docker_compose/framework_docker_injection.rs b/compiler/src/targets/mesh/docker_compose/framework_docker_injection.rs index 73ddfe64..926ae9c8 100644 --- a/compiler/src/targets/mesh/docker_compose/framework_docker_injection.rs +++ b/compiler/src/targets/mesh/docker_compose/framework_docker_injection.rs @@ -1,5 +1,6 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; +use amber_images::AMBER_ROUTER; use amber_manifest::{ProvideDecl, SlotDecl}; use amber_scenario::{ BindingEdge, BindingFrom, Component, ComponentId, Moniker, Program, ProgramMount, ProvideRef, @@ -12,8 +13,6 @@ use crate::targets::common::TargetError as MeshError; pub(crate) const FRAMEWORK_DOCKER_GATEWAY_INTERNAL_SLOT: &str = "__amber_internal_framework_docker"; pub(crate) const FRAMEWORK_DOCKER_GATEWAY_ENDPOINT: &str = "docker"; pub(crate) const FRAMEWORK_DOCKER_GATEWAY_PROVIDE: &str = "__amber_internal_framework_docker"; -pub(crate) const FRAMEWORK_DOCKER_GATEWAY_IMAGE: &str = "amber-internal://docker-gateway"; -pub(crate) const FRAMEWORK_DOCKER_GATEWAY_ENTRYPOINT: &str = "/amber-docker-gateway"; pub(crate) const FRAMEWORK_DOCKER_GATEWAY_PORT: u16 = 23750; const FRAMEWORK_DOCKER_GATEWAY_MONIKER_BASE: &str = "/__amber_internal_framework_docker_gateway"; @@ -145,9 +144,11 @@ pub(crate) fn rewrite_framework_docker_as_injected_component( } fn injected_gateway_program() -> Result { + // The injected component is a mesh identity/provide marker. Docker Compose renders only its + // router sidecar, whose router process owns the gateway runtime. serde_json::from_value(json!({ - "image": FRAMEWORK_DOCKER_GATEWAY_IMAGE, - "entrypoint": [FRAMEWORK_DOCKER_GATEWAY_ENTRYPOINT], + "image": AMBER_ROUTER.reference, + "entrypoint": ["/amber-router"], "network": { "endpoints": [ { diff --git a/compiler/src/targets/mesh/docker_compose/mod.rs b/compiler/src/targets/mesh/docker_compose/mod.rs index 73ccc92b..5d09fe73 100644 --- a/compiler/src/targets/mesh/docker_compose/mod.rs +++ b/compiler/src/targets/mesh/docker_compose/mod.rs @@ -16,9 +16,7 @@ use serde::{Deserialize, Serialize}; mod framework_docker_injection; -use framework_docker_injection::{ - FRAMEWORK_DOCKER_GATEWAY_PORT, rewrite_framework_docker_as_injected_component, -}; +use framework_docker_injection::rewrite_framework_docker_as_injected_component; use crate::{ config::analysis::ScenarioConfigAnalysis, @@ -42,8 +40,9 @@ use crate::{ }, internal_images::resolve_internal_images, mesh_config::{ - MeshConfigBuildInput, MeshConfigBuildOptions, MeshServiceName, RouterPorts, - ServiceMeshAddressing, build_mesh_config_plan, default_mesh_config_build_options, + DockerGatewayMeshConfig, MeshConfigBuildInput, MeshConfigBuildOptions, + MeshServiceName, RouterPorts, ServiceMeshAddressing, build_mesh_config_plan, + default_mesh_config_build_options, }, plan::{MeshOptions, component_label, map_program_components}, ports::{LocalRoutePorts, allocate_local_route_ports, allocate_mesh_ports}, @@ -70,7 +69,6 @@ const HELPER_INIT_SERVICE: &str = "amber-init"; const HELPER_BIN_DIR: &str = "/amber/bin"; const HELPER_BIN_PATH: &str = "/amber/bin/amber-helper"; const DOCKER_MOUNT_PROXY_SPEC_ENV: &str = "AMBER_DOCKER_MOUNT_PROXY_SPEC_B64"; -const DOCKER_GATEWAY_CONFIG_ENV: &str = "AMBER_DOCKER_GATEWAY_CONFIG_JSON"; const DOCKER_GATEWAY_HOST_SOCK_ENV: &str = "AMBER_DOCKER_SOCK"; const DOCKER_GATEWAY_CONTAINER_SOCK: &str = "/var/run/docker.sock"; const MESH_CONFIG_DIR: &str = "/amber/mesh"; @@ -319,23 +317,6 @@ impl MeshServiceName for ServiceNames { } } -#[derive(Clone, Debug, Serialize)] -struct DockerGatewayCallerConfig { - host: String, - #[serde(skip_serializing_if = "Option::is_none")] - port: Option, - component: String, - compose_service: String, -} - -#[derive(Clone, Debug, Serialize)] -struct DockerGatewayConfig { - listen: String, - docker_sock: String, - compose_project: String, - callers: Vec, -} - #[derive(Clone, Debug, Serialize)] struct DockerMountProxySpec { path: String, @@ -510,6 +491,10 @@ fn render_docker_compose_inner( router_identity_id, mesh_scope, force_router, + docker_gateway: docker_gateway_component.map(|_| DockerGatewayMeshConfig { + docker_sock: DOCKER_GATEWAY_CONTAINER_SOCK, + compose_project_env: COMPOSE_PROJECT_NAME_ENV, + }), ..default_mesh_config_build_options() }, }) @@ -803,6 +788,7 @@ fn render_docker_compose_inner( format!("AMBER_ROUTER_CONFIG_PATH={}", mesh_config_path()), format!("AMBER_ROUTER_IDENTITY_PATH={}", mesh_identity_path()), format!("AMBER_ROUTER_CONTROL_SOCKET_PATH={ROUTER_CONTROL_SOCKET_PATH_IN_CONTAINER}"), + format!("{COMPOSE_PROJECT_NAME_ENV}=${{{COMPOSE_PROJECT_NAME_ENV}:-default}}"), ]; sidecar_env_entries.extend( mesh_config_plan @@ -837,6 +823,13 @@ fn render_docker_compose_inner( sidecar_service.volumes.push(format!( "{sidecar_control_volume}:{ROUTER_CONTROL_SOCKET_DIR_IN_CONTAINER}" )); + if Some(*id) == docker_gateway_component { + sidecar_service.user = Some("0:0".to_string()); + sidecar_service.volumes.push(format!( + "${{{DOCKER_GATEWAY_HOST_SOCK_ENV}:-{}}}:{DOCKER_GATEWAY_CONTAINER_SOCK}", + detect_default_host_docker_sock().display() + )); + } let sidecar_control_init_service_name = component_control_init_service_name(&svc.sidecar); sidecar_service.depends_on = build_depends_on( false, @@ -863,7 +856,13 @@ fn render_docker_compose_inner( let mut egress_init_service = Service::new(images.helper.clone()); egress_init_service.user = Some("0:0".to_string()); - egress_init_service.command = Some(vec!["install-default-egress-guard".to_string()]); + let mesh_port = *mesh_ports_by_component + .get(id) + .expect("mesh port should be allocated before rendering services"); + egress_init_service.command = Some(vec![ + "install-network-guard".to_string(), + mesh_port.to_string(), + ]); egress_init_service.network_mode = Some(format!("service:{}", svc.sidecar)); egress_init_service.cap_add.push("NET_ADMIN".to_string()); egress_init_service.depends_on = @@ -874,13 +873,15 @@ fn render_docker_compose_inner( .services .insert(svc.egress_init.clone(), egress_init_service); + if Some(*id) == docker_gateway_component { + continue; + } + // depends_on: own sidecar + strong deps provider programs (+ amber-init for helper-backed services) let program_plan = program_plans.get(id).expect("program plan computed"); let is_site_controller = framework_component_controller_metadata(s.component(*id).metadata.as_ref()).is_some(); - let image = if Some(*id) == docker_gateway_component { - images.docker_gateway.clone() - } else if is_site_controller { + let image = if is_site_controller { images.site_controller.clone() } else { let image_plan = program_plan.image().ok_or_else(|| { @@ -933,7 +934,12 @@ fn render_docker_compose_inner( if let Some(ds) = mesh_plan.strong_deps().get(id) { for dep in ds { if let Some(dep_names) = names.get(dep) { - deps.push((dep_names.program.clone(), "service_started")); + let service = if Some(*dep) == docker_gateway_component { + dep_names.sidecar.clone() + } else { + dep_names.program.clone() + }; + deps.push((service, "service_started")); } else { return Err(format!( "internal error: missing service name for dependency {}", @@ -1027,10 +1033,6 @@ fn render_docker_compose_inner( program_service.environment = Some(Environment::List(env_entries)); } } - if Some(*id) == docker_gateway_component { - configure_injected_docker_gateway_service(&mut program_service, s, *id, svc)?; - apply_internal_service_rootfs_hardening(&mut program_service); - } for storage_mount in storage_mounts { program_service.volumes.push(format!( "{}:{}", @@ -1386,39 +1388,6 @@ fn collect_framework_docker_mount_paths( out } -fn configure_injected_docker_gateway_service( - service: &mut Service, - scenario: &Scenario, - id: ComponentId, - names: &ServiceNames, -) -> DcResult<()> { - let config_json = - build_docker_gateway_config_json(scenario.component(id).moniker.as_str(), &names.program)?; - match service.environment.take() { - Some(Environment::Map(mut env)) => { - env.insert(DOCKER_GATEWAY_CONFIG_ENV.to_string(), config_json); - service.environment = Some(Environment::Map(env)); - } - Some(Environment::List(_)) => { - return Err(DockerComposeError::Other( - "internal error: injected docker gateway should not use list-style environment" - .to_string(), - )); - } - None => { - service.environment = Some(Environment::Map(BTreeMap::from([( - DOCKER_GATEWAY_CONFIG_ENV.to_string(), - config_json, - )]))); - } - } - service.volumes.push(format!( - "${{{DOCKER_GATEWAY_HOST_SOCK_ENV}:-{}}}:{DOCKER_GATEWAY_CONTAINER_SOCK}", - detect_default_host_docker_sock().display() - )); - Ok(()) -} - fn docker_proxy_ports_by_component( scenario: &Scenario, route_ports: &LocalRoutePorts, @@ -1461,29 +1430,6 @@ fn encode_docker_mount_proxy_spec_b64( Ok(base64::engine::general_purpose::STANDARD.encode(payload)) } -fn build_docker_gateway_config_json( - caller_component: &str, - caller_compose_service: &str, -) -> DcResult { - let callers = vec![DockerGatewayCallerConfig { - host: "127.0.0.1".to_string(), - port: None, - component: caller_component.to_string(), - compose_service: caller_compose_service.to_string(), - }]; - - let config = DockerGatewayConfig { - listen: format!("0.0.0.0:{FRAMEWORK_DOCKER_GATEWAY_PORT}"), - docker_sock: DOCKER_GATEWAY_CONTAINER_SOCK.to_string(), - compose_project: "${COMPOSE_PROJECT_NAME}".to_string(), - callers, - }; - - serde_json::to_string(&config).map_err(|err| { - DockerComposeError::Other(format!("failed to serialize docker gateway config: {err}")) - }) -} - fn detect_default_host_docker_sock() -> PathBuf { #[cfg(target_os = "macos")] { diff --git a/compiler/src/targets/mesh/docker_compose/tests/mod.rs b/compiler/src/targets/mesh/docker_compose/tests/mod.rs index 1c068ce6..6d8cfa6d 100644 --- a/compiler/src/targets/mesh/docker_compose/tests/mod.rs +++ b/compiler/src/targets/mesh/docker_compose/tests/mod.rs @@ -13,7 +13,9 @@ use amber_manifest::{ CapabilityKind, FrameworkCapabilityName, Manifest, ManifestDigest, ManifestRef, Program as ManifestProgram, ProvideDecl, SlotDecl, }; -use amber_mesh::{InboundTarget, MeshProvisionOutput, MeshProvisionPlan, MeshProvisionTarget}; +use amber_mesh::{ + InboundTarget, MeshProtocol, MeshProvisionOutput, MeshProvisionPlan, MeshProvisionTarget, +}; use amber_scenario::{ BindingEdge, BindingFrom, Component, ComponentId, Moniker, ProvideRef, ResourceDecl, Scenario, ScenarioExport, SlotRef, @@ -458,16 +460,17 @@ fn assert_internal_service_rootfs_hardened(service: &super::Service, yaml: &str) assert_eq!(service.read_only, Some(true), "{yaml}"); } -fn injected_docker_gateway_service(compose: &super::DockerComposeFile) -> (&str, &super::Service) { +fn injected_docker_gateway_sidecar(compose: &super::DockerComposeFile) -> (&str, &super::Service) { compose .services .iter() .find_map(|(name, svc)| { - env_value(svc, super::DOCKER_GATEWAY_CONFIG_ENV) - .is_some() + svc.volumes + .iter() + .any(|volume| volume.ends_with(super::DOCKER_GATEWAY_CONTAINER_SOCK)) .then_some((name.as_str(), svc)) }) - .expect("injected docker gateway service missing") + .expect("injected docker gateway sidecar missing") } fn assert_depends_on(service: &super::Service, name: &str, condition: &str) { diff --git a/compiler/src/targets/mesh/docker_compose/tests/rendering.rs b/compiler/src/targets/mesh/docker_compose/tests/rendering.rs index dd8d3dc8..6cd7f2e6 100644 --- a/compiler/src/targets/mesh/docker_compose/tests/rendering.rs +++ b/compiler/src/targets/mesh/docker_compose/tests/rendering.rs @@ -363,6 +363,14 @@ fn compose_emits_otelcol_agent_and_wires_router_otel_env() { Some("service:c0-component-net"), "{yaml}" ); + assert_eq!( + egress_init.command.as_ref(), + Some(&vec![ + "install-network-guard".to_string(), + "23000".to_string() + ]), + "{yaml}" + ); assert_eq!(egress_init.user.as_deref(), Some("0:0"), "{yaml}"); assert!( egress_init.cap_add.iter().any(|cap| cap == "NET_ADMIN"), @@ -580,31 +588,17 @@ fn docker_compose_emits_gateway_for_framework_docker_binding() { "framework routing should not emit a bespoke docker gateway service" ); - let (gateway_name, gateway) = injected_docker_gateway_service(&compose); - let expected_network_mode = format!("service:{gateway_name}-net"); - assert_eq!( - gateway.network_mode.as_deref(), - Some(expected_network_mode.as_str()), - ); - assert_internal_service_rootfs_hardened(gateway, yaml.as_ref()); - - let gateway_config = env_value(gateway, super::DOCKER_GATEWAY_CONFIG_ENV) - .expect("gateway config env should be present"); - let gateway_config_json: Value = - serde_json::from_str(&gateway_config).expect("gateway config should parse"); - let callers = gateway_config_json["callers"] - .as_array() - .expect("callers should be an array"); - assert_eq!(callers.len(), 1); - assert_eq!(callers[0]["host"], "127.0.0.1"); - let gateway_component = callers[0]["component"] - .as_str() - .expect("gateway caller component should be a string"); + let (gateway_name, gateway) = injected_docker_gateway_sidecar(&compose); + assert_eq!(gateway.network_mode.as_deref(), None); + assert_eq!(gateway.user.as_deref(), Some("0:0")); assert!( - gateway_component.starts_with("/__amber_internal_framework_docker_gateway"), - "unexpected injected gateway component moniker: {gateway_component}" + gateway + .volumes + .iter() + .any(|volume| volume.ends_with(super::DOCKER_GATEWAY_CONTAINER_SOCK)), + "{yaml}" ); - assert_eq!(callers[0]["compose_service"], gateway_name); + assert_internal_service_rootfs_hardened(gateway, yaml.as_ref()); let program_service = service(&compose, "c0-component"); assert_depends_on(program_service, gateway_name, "service_started"); @@ -614,6 +608,125 @@ fn docker_compose_emits_gateway_for_framework_docker_binding() { ); } +#[test] +fn docker_compose_shared_framework_docker_gateway_preserves_consumer_identities() { + let slot_docker: SlotDecl = serde_json::from_value(json!({ "kind": "docker" })).unwrap(); + let root = Component { + id: ComponentId(0), + parent: None, + moniker: moniker("/"), + digest: digest(0), + config: None, + config_schema: None, + program: None, + slots: BTreeMap::new(), + provides: BTreeMap::new(), + resources: BTreeMap::new(), + metadata: None, + child_templates: BTreeMap::new(), + children: vec![ComponentId(1), ComponentId(2)], + }; + let child = |id: usize, name: &str| Component { + id: ComponentId(id), + parent: Some(ComponentId(0)), + moniker: moniker(&format!("/{name}")), + digest: digest(id as u8), + config: None, + config_schema: None, + program: Some(lower_test_program( + id, + json!({ + "image": "alpine:3.20", + "entrypoint": ["sh", "-lc", "sleep infinity"], + "env": { + "DOCKER_HOST": "${slots.docker.url}", + }, + }), + )), + slots: BTreeMap::from([("docker".to_string(), slot_docker.clone())]), + provides: BTreeMap::new(), + resources: BTreeMap::new(), + metadata: None, + child_templates: BTreeMap::new(), + children: Vec::new(), + }; + + let scenario = Scenario { + manifest_catalog: BTreeMap::new(), + root: ComponentId(0), + components: vec![Some(root), Some(child(1, "alpha")), Some(child(2, "beta"))], + bindings: vec![ + BindingEdge { + from: BindingFrom::Framework(amber_scenario::FrameworkRef { + authority: ComponentId(0), + capability: FrameworkCapabilityName::try_from("docker").unwrap(), + }), + to: SlotRef { + component: ComponentId(1), + name: "docker".to_string(), + }, + weak: false, + }, + BindingEdge { + from: BindingFrom::Framework(amber_scenario::FrameworkRef { + authority: ComponentId(0), + capability: FrameworkCapabilityName::try_from("docker").unwrap(), + }), + to: SlotRef { + component: ComponentId(2), + name: "docker".to_string(), + }, + weak: false, + }, + ], + exports: Vec::new(), + }; + + let output = compile_output_with_docker_feature(scenario); + let yaml = render_compose(&output).expect("compose render should succeed"); + let compose = parse_compose(&yaml); + let plan = provision_plan(&compose); + let (gateway_name, _gateway) = injected_docker_gateway_sidecar(&compose); + let gateway_target = target_for_service(&plan, gateway_name); + let gateway_inbound = gateway_target + .config + .inbound + .iter() + .find(|route| { + route.capability == super::framework_docker_injection::FRAMEWORK_DOCKER_GATEWAY_PROVIDE + }) + .expect("gateway inbound docker route"); + assert_eq!(gateway_inbound.protocol, MeshProtocol::Tcp); + match &gateway_inbound.target { + amber_mesh::InboundTarget::DockerGateway { + docker_sock, + compose_project_env, + } => { + assert_eq!(docker_sock, Path::new(super::DOCKER_GATEWAY_CONTAINER_SOCK)); + assert_eq!(compose_project_env, super::COMPOSE_PROJECT_NAME_ENV); + } + other => panic!("unexpected docker gateway target: {other:?}"), + } + assert_eq!(gateway_inbound.allowed_issuers.len(), 2); + + for service_name in ["c1-alpha", "c2-beta"] { + let program_service = service(&compose, service_name); + assert_depends_on(program_service, gateway_name, "service_started"); + assert_eq!( + env_value(program_service, "DOCKER_HOST").as_deref(), + Some("tcp://127.0.0.1:20000"), + ); + let target = target_for_service(&plan, &format!("{service_name}-net")); + let outbound = target + .config + .outbound + .iter() + .find(|route| route.slot == "docker") + .expect("consumer outbound docker route"); + assert_eq!(outbound.protocol, MeshProtocol::Tcp); + } +} + #[test] fn docker_compose_emits_framework_docker_mount_proxy_wiring() { let program = lower_test_program( @@ -666,12 +779,9 @@ fn docker_compose_emits_framework_docker_mount_proxy_wiring() { "framework routing should not emit a bespoke docker gateway service" ); - let (gateway_name, gateway) = injected_docker_gateway_service(&compose); - let expected_network_mode = format!("service:{gateway_name}-net"); - assert_eq!( - gateway.network_mode.as_deref(), - Some(expected_network_mode.as_str()), - ); + let (gateway_name, gateway) = injected_docker_gateway_sidecar(&compose); + assert_eq!(gateway.network_mode.as_deref(), None); + assert_internal_service_rootfs_hardened(gateway, yaml.as_ref()); let program_service = service(&compose, "c0-component"); let entrypoint = program_service diff --git a/compiler/src/targets/mesh/internal_images.rs b/compiler/src/targets/mesh/internal_images.rs index 1b417162..169a0735 100644 --- a/compiler/src/targets/mesh/internal_images.rs +++ b/compiler/src/targets/mesh/internal_images.rs @@ -1,5 +1,5 @@ use amber_images::{ - AMBER_DOCKER_GATEWAY, AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER, AMBER_SITE_CONTROLLER, + AMBER_HELPER, AMBER_PROVISIONER, AMBER_ROUTER, AMBER_SITE_CONTROLLER, INTERNAL_IMAGE_OVERRIDE_KEYS, ImageRef, override_reference, parse_dev_image_tag_overrides, }; @@ -8,7 +8,6 @@ pub(crate) struct InternalImages { pub(crate) helper: String, pub(crate) provisioner: String, pub(crate) router: String, - pub(crate) docker_gateway: String, pub(crate) site_controller: String, } @@ -17,7 +16,6 @@ pub(crate) fn resolve_internal_images() -> Result { helper: default_reference(&AMBER_HELPER), provisioner: default_reference(&AMBER_PROVISIONER), router: default_reference(&AMBER_ROUTER), - docker_gateway: default_reference(&AMBER_DOCKER_GATEWAY), site_controller: default_reference(&AMBER_SITE_CONTROLLER), }; @@ -26,9 +24,6 @@ pub(crate) fn resolve_internal_images() -> Result { "router" => images.router = override_reference(&AMBER_ROUTER, &value), "helper" => images.helper = override_reference(&AMBER_HELPER, &value), "provisioner" => images.provisioner = override_reference(&AMBER_PROVISIONER, &value), - "docker_gateway" => { - images.docker_gateway = override_reference(&AMBER_DOCKER_GATEWAY, &value) - } "site_controller" => { images.site_controller = override_reference(&AMBER_SITE_CONTROLLER, &value) } diff --git a/compiler/src/targets/mesh/kubernetes/tests/mod.rs b/compiler/src/targets/mesh/kubernetes/tests/mod.rs index 52406787..360626b7 100644 --- a/compiler/src/targets/mesh/kubernetes/tests/mod.rs +++ b/compiler/src/targets/mesh/kubernetes/tests/mod.rs @@ -1,4 +1,5 @@ use std::{ + collections::BTreeSet, fs, net::SocketAddr, path::{Path, PathBuf}, diff --git a/compiler/src/targets/mesh/kubernetes/tests/rendering.rs b/compiler/src/targets/mesh/kubernetes/tests/rendering.rs index 56f97768..de767309 100644 --- a/compiler/src/targets/mesh/kubernetes/tests/rendering.rs +++ b/compiler/src/targets/mesh/kubernetes/tests/rendering.rs @@ -280,6 +280,114 @@ fn kubernetes_emits_router_for_external_slots() { ); } +#[test] +fn kubernetes_network_policy_only_exposes_component_mesh_port() { + let dir = tempdir().expect("temp dir"); + let root_path = dir.path().join("root.json5"); + let server_path = dir.path().join("server.json5"); + let client_path = dir.path().join("client.json5"); + + fs::write( + &root_path, + format!( + r##" + {{ + manifest_version: "0.1.0", + components: {{ + server: "{}", + client: "{}" + }}, + bindings: [ + {{ to: "#client.api", from: "#server.api" }} + ] + }} + "##, + file_url(&server_path), + file_url(&client_path), + ), + ) + .expect("write root manifest"); + + fs::write( + &server_path, + r#" + { + manifest_version: "0.1.0", + program: { + image: "server", + entrypoint: ["server"], + network: { + endpoints: [{ name: "api", port: 8080, protocol: "http" }] + } + }, + provides: { api: { kind: "http", endpoint: "api" } }, + exports: { api: "api" } + } + "#, + ) + .expect("write server manifest"); + fs::write( + &client_path, + r#" + { + manifest_version: "0.1.0", + program: { + image: "client", + entrypoint: ["client"], + env: { API_URL: "${slots.api.url}" } + }, + slots: { api: { kind: "http" } } + } + "#, + ) + .expect("write client manifest"); + + let compiler = Compiler::new(Resolver::new(), DigestStore::default()); + let opts = CompileOptions { + optimize: OptimizeOptions { dce: false }, + ..Default::default() + }; + let rt = tokio::runtime::Runtime::new().expect("tokio runtime"); + let output = rt + .block_on(compiler.compile(ManifestRef::from_url(file_url(&root_path)), opts)) + .expect("compile scenario"); + let artifact = render_artifact(&output); + + let service_yaml = artifact + .files + .get(&PathBuf::from("04-services/c2-server.yaml")) + .expect("server service"); + let service_doc: serde_yaml::Value = + serde_yaml::from_str(service_yaml).expect("parse server service"); + let service_ports = service_doc["spec"]["ports"] + .as_sequence() + .expect("service ports") + .iter() + .map(|port| port["port"].as_u64().expect("service port")) + .collect::>(); + assert_eq!(service_ports, BTreeSet::from([23000])); + + let netpol_yaml = artifact + .files + .get(&PathBuf::from("05-networkpolicies/c2-server-netpol.yaml")) + .expect("server network policy"); + let netpol_doc: serde_yaml::Value = + serde_yaml::from_str(netpol_yaml).expect("parse server network policy"); + let ingress_ports = netpol_doc["spec"]["ingress"] + .as_sequence() + .expect("ingress rules") + .iter() + .flat_map(|rule| rule["ports"].as_sequence().into_iter().flatten()) + .map(|port| port["port"].as_u64().expect("network policy port")) + .collect::>(); + assert_eq!(ingress_ports, BTreeSet::from([23000])); + assert!( + !netpol_yaml.contains("8080"), + "program endpoint port must not be directly reachable through \ + NetworkPolicy:\n{netpol_yaml}" + ); +} + #[test] fn kubernetes_storage_mounts_emit_pvc_and_recreate_deployment() { let fixture_dir = tempdir().expect("fixture dir"); diff --git a/compiler/src/targets/mesh/mesh_config.rs b/compiler/src/targets/mesh/mesh_config.rs index 66226be3..326cbe14 100644 --- a/compiler/src/targets/mesh/mesh_config.rs +++ b/compiler/src/targets/mesh/mesh_config.rs @@ -1,6 +1,6 @@ use std::collections::{BTreeSet, HashMap}; -use amber_manifest::NetworkProtocol; +use amber_manifest::{CapabilityKind, NetworkProtocol}; use amber_mesh::{ InboundRoute, InboundTarget, MeshConfigTemplate, MeshIdentityTemplate, MeshPeerTemplate, MeshProtocol, OutboundRoute, component_route_id, http_route_plugins_for_capability_kind, @@ -31,6 +31,13 @@ pub(crate) struct MeshConfigBuildOptions<'a> { pub(crate) router_mesh_listen_addr: &'a str, pub(crate) router_control_listen_addr: &'a str, pub(crate) force_router: bool, + pub(crate) docker_gateway: Option>, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct DockerGatewayMeshConfig<'a> { + pub(crate) docker_sock: &'a str, + pub(crate) compose_project_env: &'a str, } pub(crate) fn default_mesh_config_build_options() -> MeshConfigBuildOptions<'static> { @@ -41,6 +48,7 @@ pub(crate) fn default_mesh_config_build_options() -> MeshConfigBuildOptions<'sta router_mesh_listen_addr: "0.0.0.0", router_control_listen_addr: "0.0.0.0", force_router: false, + docker_gateway: None, } } @@ -251,6 +259,7 @@ pub(crate) fn build_mesh_config_plan( let Some(endpoint) = endpoint else { continue; }; + let capability_kind = provide_decl.decl.kind.as_str(); let protocol = mesh_protocol(endpoint.protocol)?; let mut issuers: BTreeSet = BTreeSet::new(); @@ -294,6 +303,24 @@ pub(crate) fn build_mesh_config_plan( continue; } + let target = if capability_kind == CapabilityKind::Docker.as_str() { + let docker_gateway = input.options.docker_gateway.ok_or_else(|| { + MeshError::new(format!( + "mesh config for {}.{} requires docker gateway runtime configuration", + component_label(scenario, id), + provide_name + )) + })?; + InboundTarget::DockerGateway { + docker_sock: docker_gateway.docker_sock.into(), + compose_project_env: docker_gateway.compose_project_env.to_string(), + } + } else { + InboundTarget::Local { + port: endpoint.port, + } + }; + inbound.push(InboundRoute { route_id: component_route_id(&identity.id, provide_name, protocol), capability: provide_name.clone(), @@ -304,9 +331,7 @@ pub(crate) fn build_mesh_config_plan( Some(provide_decl.decl.kind.as_str()), protocol, ), - target: InboundTarget::Local { - port: endpoint.port, - }, + target, allowed_issuers: issuers.into_iter().collect(), }); } @@ -329,12 +354,12 @@ pub(crate) fn build_mesh_config_plan( .expect("provider identity missing") .id .clone(); - let protocol = mesh_protocol(binding.endpoint.protocol)?; let provide_decl = scenario .component(binding.provider) .provides .get(&binding.provide) .expect("binding provide should exist"); + let protocol = mesh_protocol(binding.endpoint.protocol)?; outbound.push(OutboundRoute { route_id: component_route_id(&peer_id, &binding.provide, protocol), rewrite_route_id: None, @@ -517,12 +542,12 @@ pub(crate) fn build_mesh_config_plan( .expect("export provider identity missing") .id .clone(); - let protocol = mesh_protocol(export.endpoint.protocol)?; let provide_decl = scenario .component(export.provider) .provides .get(&export.provide) .expect("export provide should exist"); + let protocol = mesh_protocol(export.endpoint.protocol)?; let provider_route_id = component_route_id(&peer_id, &export.provide, protocol); inbound.push(InboundRoute { route_id: router_export_route_id(&export.name, protocol), diff --git a/compiler/src/targets/vm/mod.rs b/compiler/src/targets/vm/mod.rs index 516ea676..881df328 100644 --- a/compiler/src/targets/vm/mod.rs +++ b/compiler/src/targets/vm/mod.rs @@ -31,7 +31,7 @@ use crate::{ }, mesh_config::{ MeshAddressing, MeshConfigBuildInput, MeshConfigBuildOptions, RouterPorts, - build_mesh_config_plan, + build_mesh_config_plan, default_mesh_config_build_options, }, plan::{MeshOptions, MeshPlan, build_mesh_plan, map_program_components}, ports::placeholder_local_route_ports, @@ -331,6 +331,7 @@ pub(crate) fn emit_vm_artifact_with_options( router_mesh_listen_addr: "127.0.0.1", router_control_listen_addr: "127.0.0.1", force_router: options.force_router, + ..default_mesh_config_build_options() }, })?; diff --git a/docker/amber-cli/Dockerfile b/docker/amber-cli/Dockerfile index 5da6ddea..c4359cb7 100644 --- a/docker/amber-cli/Dockerfile +++ b/docker/amber-cli/Dockerfile @@ -50,13 +50,14 @@ COPY runtime/provisioner/Cargo.toml runtime/provisioner/ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/router/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src runtime/provisioner/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/lib.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ cargo fetch --locked COPY cli ./cli COPY compiler ./compiler COPY examples ./examples COPY images ./images +COPY runtime/docker-gateway ./runtime/docker-gateway COPY runtime/helper ./runtime/helper COPY runtime/mesh ./runtime/mesh COPY runtime/proxy ./runtime/proxy diff --git a/docker/amber-docker-gateway/Dockerfile b/docker/amber-docker-gateway/Dockerfile deleted file mode 100644 index a08f9988..00000000 --- a/docker/amber-docker-gateway/Dockerfile +++ /dev/null @@ -1,75 +0,0 @@ -# syntax=docker/dockerfile:1.12 - -FROM rust:1.93.1-slim-trixie AS base-builder - -WORKDIR /app - -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ - apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - g++ \ - musl-tools \ - pkg-config - -COPY rust-toolchain.toml . -RUN rustup show -ARG TARGETARCH -RUN case "${TARGETARCH}" in \ - amd64) target="x86_64-unknown-linux-musl" ;; \ - arm64) target="aarch64-unknown-linux-musl" ;; \ - *) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ - esac && \ - echo "${target}" > /tmp/rust-target && \ - rustup target add "${target}" - -FROM base-builder AS builder - -# Prefetch Rust dependencies for better Docker layer caching. -COPY Cargo.toml Cargo.lock ./ -COPY cli/Cargo.toml cli/ -COPY compiler/config/Cargo.toml compiler/config/ -COPY compiler/Cargo.toml compiler/ -COPY runtime/docker-gateway/Cargo.toml runtime/docker-gateway/ -COPY runtime/helper/Cargo.toml runtime/helper/ -COPY images/Cargo.toml images/ -COPY manager/Cargo.toml manager/ -COPY compiler/json5/Cargo.toml compiler/json5/ -COPY compiler/manifest/Cargo.toml compiler/manifest/ -COPY runtime/mesh/Cargo.toml runtime/mesh/ -COPY runtime/proxy/Cargo.toml runtime/proxy/ -COPY runtime/provisioner/Cargo.toml runtime/provisioner/ -COPY runtime/router/Cargo.toml runtime/router/ -COPY runtime/site-controller/Cargo.toml runtime/site-controller/ -COPY compiler/resolver/Cargo.toml compiler/resolver/ -COPY compiler/scenario/Cargo.toml compiler/scenario/ -COPY compiler/template/Cargo.toml compiler/template/ - -RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ - --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ - mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/provisioner/src runtime/router/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/provisioner/src/main.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs && \ - cargo fetch --locked - -COPY runtime/docker-gateway ./runtime/docker-gateway - -ARG BUILD_MODE=release -RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ - --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ - --mount=type=cache,target=/app/target,sharing=locked \ - target=$(cat /tmp/rust-target) && \ - if [ "$BUILD_MODE" = "release" ]; then \ - cargo build --locked --release -p amber-docker-gateway --target "${target}"; \ - build_dir=release; \ - else \ - cargo build --locked -p amber-docker-gateway --target "${target}"; \ - build_dir=debug; \ - fi && \ - install -D -m 0755 /app/target/"${target}"/"${build_dir}"/amber-docker-gateway /out/amber-docker-gateway - -FROM gcr.io/distroless/static-debian13 AS runtime - -COPY --from=builder /out/amber-docker-gateway /amber-docker-gateway - -ENTRYPOINT ["/amber-docker-gateway"] diff --git a/docker/amber-helper/Dockerfile b/docker/amber-helper/Dockerfile index ea68f719..2a854445 100644 --- a/docker/amber-helper/Dockerfile +++ b/docker/amber-helper/Dockerfile @@ -49,7 +49,7 @@ COPY runtime/provisioner/Cargo.toml runtime/provisioner/ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/router/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src runtime/provisioner/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/lib.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ cargo fetch --locked COPY compiler ./compiler diff --git a/docker/amber-manager/Dockerfile b/docker/amber-manager/Dockerfile index af669fb8..da93a0ed 100644 --- a/docker/amber-manager/Dockerfile +++ b/docker/amber-manager/Dockerfile @@ -49,7 +49,7 @@ COPY compiler/template/Cargo.toml compiler/template/ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/provisioner/src runtime/router/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/provisioner/src/main.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs && \ + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/lib.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/provisioner/src/main.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs && \ cargo fetch --locked COPY compiler ./compiler diff --git a/docker/amber-provisioner/Dockerfile b/docker/amber-provisioner/Dockerfile index 9128b491..76b67f9a 100644 --- a/docker/amber-provisioner/Dockerfile +++ b/docker/amber-provisioner/Dockerfile @@ -44,7 +44,7 @@ COPY compiler/scenario/Cargo.toml compiler/scenario/ COPY compiler/template/Cargo.toml compiler/template/ RUN mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/provisioner/src runtime/router/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/provisioner/src/main.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/lib.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/provisioner/src/main.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs RUN cargo fetch --locked COPY compiler ./compiler diff --git a/docker/amber-router/Dockerfile b/docker/amber-router/Dockerfile index b52d9717..d9296745 100644 --- a/docker/amber-router/Dockerfile +++ b/docker/amber-router/Dockerfile @@ -49,10 +49,11 @@ COPY runtime/provisioner/Cargo.toml runtime/provisioner/ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/router/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src runtime/provisioner/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/lib.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ cargo fetch --locked COPY compiler ./compiler +COPY runtime/docker-gateway ./runtime/docker-gateway COPY runtime/mesh ./runtime/mesh COPY runtime/router ./runtime/router diff --git a/docker/amber-site-controller/Dockerfile b/docker/amber-site-controller/Dockerfile index 33169dc2..fdd9907a 100644 --- a/docker/amber-site-controller/Dockerfile +++ b/docker/amber-site-controller/Dockerfile @@ -49,13 +49,14 @@ COPY runtime/provisioner/Cargo.toml runtime/provisioner/ RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git,sharing=locked \ mkdir -p cli/src compiler/config/src compiler/src runtime/docker-gateway/src runtime/helper/src images/src manager/src compiler/json5/src compiler/manifest/src runtime/mesh/src runtime/proxy/src runtime/router/src runtime/site-controller/src runtime/site-controller/src/bin compiler/resolver/src compiler/scenario/src compiler/template/src runtime/provisioner/src && \ - touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/main.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ + touch cli/src/main.rs compiler/config/src/lib.rs compiler/src/lib.rs runtime/docker-gateway/src/lib.rs runtime/helper/src/main.rs images/src/lib.rs manager/src/lib.rs manager/src/main.rs compiler/json5/src/lib.rs compiler/manifest/src/lib.rs runtime/mesh/src/lib.rs runtime/proxy/src/lib.rs runtime/router/src/main.rs runtime/site-controller/src/lib.rs runtime/site-controller/src/bin/amber-site-controller.rs compiler/resolver/src/lib.rs compiler/scenario/src/lib.rs compiler/template/src/lib.rs runtime/provisioner/src/main.rs && \ cargo fetch --locked COPY cli ./cli COPY compiler ./compiler COPY examples ./examples COPY images ./images +COPY runtime/docker-gateway ./runtime/docker-gateway COPY runtime/helper ./runtime/helper COPY runtime/mesh ./runtime/mesh COPY runtime/proxy ./runtime/proxy diff --git a/docker/images.json b/docker/images.json index 43dca0d5..759f98a9 100644 --- a/docker/images.json +++ b/docker/images.json @@ -19,12 +19,6 @@ "dockerfile": "docker/amber-router/Dockerfile", "version": "v0.2.x" }, - { - "name": "amber-docker-gateway", - "context": ".", - "dockerfile": "docker/amber-docker-gateway/Dockerfile", - "version": "v0.1.x" - }, { "name": "amber-provisioner", "context": ".", diff --git a/images/src/lib.rs b/images/src/lib.rs index 46fa81c8..9db30abe 100644 --- a/images/src/lib.rs +++ b/images/src/lib.rs @@ -6,13 +6,8 @@ use std::{ pub mod versioning; pub const DEV_IMAGE_TAGS_ENV: &str = "AMBER_DEV_IMAGE_TAGS"; -pub const INTERNAL_IMAGE_OVERRIDE_KEYS: &[&str] = &[ - "router", - "helper", - "provisioner", - "docker_gateway", - "site_controller", -]; +pub const INTERNAL_IMAGE_OVERRIDE_KEYS: &[&str] = + &["router", "helper", "provisioner", "site_controller"]; #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] pub struct ImageRef { diff --git a/runtime/docker-gateway/Cargo.toml b/runtime/docker-gateway/Cargo.toml index d2ad1a83..42077392 100644 --- a/runtime/docker-gateway/Cargo.toml +++ b/runtime/docker-gateway/Cargo.toml @@ -3,7 +3,6 @@ name = "amber-docker-gateway" edition = "2024" [dependencies] -base64 = { workspace = true } bytes = "1.10.1" dashmap = "6.1.0" http-body-util = "0.1.3" @@ -15,7 +14,7 @@ serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } serde_with = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net", "io-util", "sync", "time", "signal"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net", "io-util", "sync", "time"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } url = { workspace = true } diff --git a/runtime/docker-gateway/README.md b/runtime/docker-gateway/README.md index 1021ac5b..49b3c9b2 100644 --- a/runtime/docker-gateway/README.md +++ b/runtime/docker-gateway/README.md @@ -1,82 +1,59 @@ # Amber Docker Gateway -Amber Docker Gateway is a small policy layer that sits between Amber components and the host -Docker Engine API. It accepts TCP connections (for example from `uds2tcp`), authenticates callers -based on the peer address, injects Compose and ownership labels on create, and scopes all -operations to the owning component inside the scenario. +Amber Docker Gateway is the Docker Engine policy layer used by Amber router sidecars for +`framework.docker` capabilities. It is a Rust library, not a standalone network service: the +router accepts a Noise-authenticated mesh stream, uses the authenticated mesh peer id as the caller +identity, and hands the raw TCP stream to the gateway handler. -The gateway is designed to run as an Amber component injected by the compiler. It behaves like a -normal component container, except that it receives a host `docker.sock` mount. +There is deliberately no source-IP, source-port, or caller-supplied-header authentication path. +All Amber component communication to the gateway travels over the mesh. The host Docker socket is +mounted only into the injected gateway sidecar, and the socket peer address is not part of the +authorization model. -## What it does +## What It Does -- Authenticates each TCP connection using the peer IP (and optional port) resolved from caller - host entries in config. +- Uses the mesh peer id supplied by the router as the owning component identity. - Injects ownership labels and `com.docker.compose.project` on create so Compose can clean up - with `--remove-orphans`. + resources with `--remove-orphans`. - Enforces per-component scoping: only the owning component can inspect, modify, exec, or delete its containers, networks, and volumes. -- Blocks endpoints that cannot be scoped safely (default-deny). -- Forwards all allowed requests to the upstream Docker daemon over a Unix socket using - `hyperlocal`. -- Supports HTTP upgrade/hijack (exec/attach). +- Blocks endpoints that cannot be scoped safely. +- Forwards allowed requests to the upstream Docker daemon over a Unix socket. +- Supports HTTP upgrade/hijack for Docker exec and attach flows. ## Configuration -The gateway reads configuration from one of these environment variables: - -- `AMBER_DOCKER_GATEWAY_CONFIG_B64` (base64 JSON) -- `AMBER_DOCKER_GATEWAY_CONFIG_JSON` (raw JSON) - -Example config: +The router creates `DockerGatewayRuntime` from: ```json { - "listen": "0.0.0.0:23750", "docker_sock": "/var/run/docker.sock", - "compose_project": "amber-scenario-123", - "callers": [ - { - "host": "c2-client-net", - "port": 41732, - "component": "component-a", - "compose_service": "c2-client-net" - } - ] + "compose_project": "amber-scenario-123" } ``` -Notes: - -- `compose_project` is required. It is the Docker Compose project name for the scenario. -- `callers` is required. Each entry maps a peer `host` (and optional `port`) to a component and - its Compose service name. The gateway resolves caller hosts periodically and authenticates peers - by the resolved source IP (plus optional source port). -- If your TCP proxy uses ephemeral source ports, omit `port` and match by resolved IP only. -- Ownership labels are fixed to `amber.component` and `amber.project`. +`docker_sock` is the Docker daemon Unix socket inside the sidecar. `compose_project` is the +scenario's Docker Compose project name and is used for Docker resource scoping. -## Labels and scoping +## Labels and Scoping On create, the gateway injects these labels: -- Ownership labels: `amber.component`, `amber.project` -- Compose project label: `com.docker.compose.project` (normalized to outer scenario project) +- `amber.component` +- `amber.project` +- `com.docker.compose.project` -These labels are used for two purposes: +The Amber labels enforce ownership. The Compose project label keeps resources created through the +gateway in the scenario's Compose project so normal Compose teardown removes them. -1. Compose grouping (`com.docker.compose.project`) so `docker compose --remove-orphans` - works for resources created through the gateway. -2. Ownership enforcement so only the owning component and scenario project can act on the resource. - -## Allowed API surface +## Allowed API Surface The gateway is default-deny. It only allows endpoints that can be scoped safely: -- `/containers/create`, `/networks/create`, `/volumes/create` (labels injected) -- `/images/{name}/json` (read-only image inspect; required by `docker compose up --no-build`) -- Container-scoped endpoints (start/stop/logs/inspect/remove/exec/etc) after ownership check -- Exec endpoints (`/exec/{id}/start`, `/exec/{id}/json`, `/exec/{id}/resize`) after mapping - exec ID to container ownership +- `/containers/create`, `/networks/create`, `/volumes/create` +- `/images/{name}/json` +- Container-scoped endpoints after ownership checks +- Exec endpoints after mapping exec ID to container ownership - List/prune endpoints with label filters injected: - `/containers/json`, `/events`, `/containers/prune` - `/networks`, `/networks/prune` @@ -86,16 +63,3 @@ Explicitly blocked: - `/networks/{id}/connect` and `/disconnect` - Any other endpoint that cannot be scoped to a single component - -If you need to allow a specific endpoint, update the policy in the handler to explicitly -authorize it. - -## Running locally (example) - -```sh -export AMBER_DOCKER_GATEWAY_CONFIG_JSON='{"listen":"0.0.0.0:23750","docker_sock":"/var/run/docker.sock","compose_project":"dev","callers":[{"host":"localhost","component":"dev","compose_service":"dev"}]}' -./target/debug/amber-docker-gateway -``` - -In production, the component should mount the host Docker socket at `/var/run/docker.sock` and -receive the config via environment variables. diff --git a/runtime/docker-gateway/src/lib.rs b/runtime/docker-gateway/src/lib.rs index 672ecbb9..208df83c 100644 --- a/runtime/docker-gateway/src/lib.rs +++ b/runtime/docker-gateway/src/lib.rs @@ -1,13 +1,10 @@ use std::{ collections::{HashMap, HashSet}, - env, - net::{IpAddr, SocketAddr}, path::PathBuf, sync::Arc, time::Duration, }; -use base64::Engine as _; use bytes::Bytes; use http_body_util::{BodyExt, Full, combinators::BoxBody}; use hyper::{ @@ -28,19 +25,16 @@ use serde::{Deserialize, de::DeserializeOwned}; use serde_with::{DefaultOnNull, serde_as}; use thiserror::Error; use tokio::{ - net::{TcpListener, UnixStream}, - sync::RwLock, - time::{sleep, timeout}, + io::{AsyncRead, AsyncWrite}, + net::UnixStream, + time::timeout, }; -const CONFIG_B64_ENV: &str = "AMBER_DOCKER_GATEWAY_CONFIG_B64"; -const CONFIG_JSON_ENV: &str = "AMBER_DOCKER_GATEWAY_CONFIG_JSON"; const COMPOSE_PROJECT_LABEL: &str = "com.docker.compose.project"; #[cfg(test)] const COMPOSE_SERVICE_LABEL: &str = "com.docker.compose.service"; const AMBER_COMPONENT_LABEL: &str = "amber.component"; const AMBER_PROJECT_LABEL: &str = "amber.project"; -const CALLER_REFRESH_INTERVAL: Duration = Duration::from_secs(5); const SHUTDOWN_CLEANUP_TIMEOUT: Duration = Duration::from_secs(8); const EXEC_CACHE_TTL: Duration = Duration::from_secs(5 * 60); const EXEC_CACHE_MAX_ENTRIES: u64 = 8_192; @@ -49,55 +43,28 @@ pub type BoxError = Box; pub type ProxyBody = BoxBody; type GatewayResult = Result>>; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ShutdownReason { - Interrupt, - Terminated, -} - #[derive(Debug, Error)] pub enum DockerGatewayError { - #[error("missing docker gateway config (set {CONFIG_B64_ENV} or {CONFIG_JSON_ENV})")] - MissingConfig, #[error("invalid docker gateway config: {0}")] InvalidConfig(String), - #[error("failed to bind {addr}: {source}")] - BindFailed { - addr: SocketAddr, - #[source] - source: std::io::Error, - }, } #[derive(Clone, Debug, Deserialize)] pub struct DockerGatewayConfig { - pub listen: SocketAddr, pub docker_sock: PathBuf, pub compose_project: String, - pub callers: Vec, -} - -#[derive(Clone, Debug, Deserialize)] -pub struct CallerConfig { - pub host: String, - #[serde(default)] - pub port: Option, - pub component: String, - pub compose_service: String, } #[derive(Clone)] struct ConnState { state: Arc, - peer: SocketAddr, - identity: Option, + identity: CallerIdentity, } struct State { cfg: Arc, client: Client, exec_map: Cache, - callers_by_ip: RwLock>>, } #[derive(Clone, Debug)] @@ -111,12 +78,6 @@ struct CallerIdentity { component: String, } -#[derive(Clone, Debug)] -struct ResolvedCaller { - component: String, - port: Option, -} - #[derive(Debug, Deserialize)] struct ContainerInspectResponse { #[serde(rename = "Config")] @@ -279,63 +240,58 @@ struct NetworkingConfigRequest { } impl DockerGatewayConfig { - pub fn from_env() -> Result { - if let Ok(b64) = env::var(CONFIG_B64_ENV) { - if b64.trim().is_empty() { - return Err(DockerGatewayError::MissingConfig); - } - let decoded = base64::engine::general_purpose::STANDARD - .decode(b64.as_bytes()) - .map_err(|err| DockerGatewayError::InvalidConfig(err.to_string()))?; - let parsed = serde_json::from_slice(&decoded) - .map_err(|err| DockerGatewayError::InvalidConfig(err.to_string()))?; - return Self::validate(parsed); - } - - if let Ok(raw) = env::var(CONFIG_JSON_ENV) { - if raw.trim().is_empty() { - return Err(DockerGatewayError::MissingConfig); - } - let parsed = serde_json::from_str(&raw) - .map_err(|err| DockerGatewayError::InvalidConfig(err.to_string()))?; - return Self::validate(parsed); - } - - Err(DockerGatewayError::MissingConfig) - } - - fn validate(config: Self) -> Result { + pub fn validate(config: Self) -> Result { if config.compose_project.trim().is_empty() { return Err(DockerGatewayError::InvalidConfig( "compose_project must not be empty".to_string(), )); } - if config.callers.is_empty() { - return Err(DockerGatewayError::InvalidConfig( - "callers must not be empty".to_string(), - )); - } - for caller in &config.callers { - if caller.host.trim().is_empty() { - return Err(DockerGatewayError::InvalidConfig( - "caller host must not be empty".to_string(), - )); - } - if caller.component.trim().is_empty() { - return Err(DockerGatewayError::InvalidConfig( - "caller component must not be empty".to_string(), - )); - } - if caller.compose_service.trim().is_empty() { - return Err(DockerGatewayError::InvalidConfig( - "caller compose_service must not be empty".to_string(), - )); - } - } Ok(config) } } +#[derive(Clone)] +pub struct DockerGatewayRuntime { + state: Arc, +} + +impl DockerGatewayRuntime { + pub fn new(config: DockerGatewayConfig) -> Result { + Ok(Self { + state: Arc::new(State::new(DockerGatewayConfig::validate(config)?)), + }) + } + + pub async fn serve_connection(&self, stream: S, peer_id: String) -> Result<(), BoxError> + where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, + { + let conn_state = Arc::new(ConnState { + state: self.state.clone(), + identity: CallerIdentity { component: peer_id }, + }); + let svc = service_fn(move |req: Request| { + let conn_state = conn_state.clone(); + async move { + let req = req.map(box_body_from_incoming); + Ok::<_, std::convert::Infallible>(handle(req, conn_state).await) + } + }); + + http1::Builder::new() + .preserve_header_case(true) + .title_case_headers(true) + .serve_connection(TokioIo::new(stream), svc) + .with_upgrades() + .await + .map_err(|err| -> BoxError { Box::new(err) }) + } + + pub async fn cleanup_created_resources(&self) { + self.state.cleanup_created_resources().await; + } +} + impl State { fn new(config: DockerGatewayConfig) -> Self { let connector = UnixConnector; @@ -347,69 +303,9 @@ impl State { .time_to_live(EXEC_CACHE_TTL) .max_capacity(EXEC_CACHE_MAX_ENTRIES) .build(), - callers_by_ip: RwLock::new(HashMap::new()), } } - async fn refresh_callers(&self) { - let mut callers_by_ip: HashMap> = HashMap::new(); - - for caller in &self.cfg.callers { - match tokio::net::lookup_host((caller.host.as_str(), 0)).await { - Ok(addrs) => { - let mut saw_address = false; - for addr in addrs { - saw_address = true; - callers_by_ip - .entry(addr.ip()) - .or_default() - .push(ResolvedCaller { - component: caller.component.clone(), - port: caller.port, - }); - } - - if !saw_address { - tracing::warn!( - "docker gateway caller resolution returned no addresses for host {}", - caller.host - ); - } - } - Err(err) => { - tracing::warn!( - "docker gateway caller resolution failed for host {}: {err}", - caller.host - ); - } - } - } - - *self.callers_by_ip.write().await = callers_by_ip; - } - - async fn resolve_identity(&self, peer: SocketAddr) -> Option { - let callers = self.callers_by_ip.read().await; - let matches = callers.get(&peer.ip())?; - - let mut identity: Option<&str> = None; - for entry in matches { - if entry.port.is_some_and(|port| port != peer.port()) { - continue; - } - - match identity { - None => identity = Some(entry.component.as_str()), - Some(component) if component == entry.component.as_str() => {} - Some(_) => return None, - } - } - - identity.map(|component| CallerIdentity { - component: component.to_string(), - }) - } - async fn cleanup_created_resources(&self) { if timeout( SHUTDOWN_CLEANUP_TIMEOUT, @@ -499,108 +395,8 @@ impl State { } } -async fn resolve_connection_identity(state: &State, peer: SocketAddr) -> Option { - if let Some(identity) = state.resolve_identity(peer).await { - return Some(identity); - } - // Newly started compose networks can make caller hostnames resolvable a moment after startup. - // Refresh once on a miss so first requests do not fail with a transient unauthorized error. - state.refresh_callers().await; - state.resolve_identity(peer).await -} - -pub async fn run(config: DockerGatewayConfig) -> Result<(), DockerGatewayError> { - let state = Arc::new(State::new(config)); - state.refresh_callers().await; - - let refresh_state = state.clone(); - let refresh_task = tokio::spawn(async move { - loop { - sleep(CALLER_REFRESH_INTERVAL).await; - refresh_state.refresh_callers().await; - } - }); - - let listener = TcpListener::bind(state.cfg.listen) - .await - .map_err(|source| DockerGatewayError::BindFailed { - addr: state.cfg.listen, - source, - })?; - - let shutdown = shutdown_signal(); - tokio::pin!(shutdown); - - loop { - tokio::select! { - reason = &mut shutdown => { - refresh_task.abort(); - match reason { - ShutdownReason::Interrupt => { - state.cleanup_created_resources().await; - } - ShutdownReason::Terminated => { - // `docker compose down` sends SIGTERM and also performs its own teardown. - // Skipping gateway cleanup avoids duplicate deletes and "No such container" races. - tracing::warn!( - "docker gateway received SIGTERM; skipping shutdown cleanup to avoid \ - teardown races" - ); - } - } - return Ok(()); - } - accepted = listener.accept() => { - let (stream, peer) = match accepted { - Ok(value) => value, - Err(err) => { - tracing::warn!("docker gateway accept failed: {err}"); - continue; - } - }; - - let conn_state = Arc::new(ConnState { - state: state.clone(), - peer, - identity: resolve_connection_identity(state.as_ref(), peer).await, - }); - - let io = TokioIo::new(stream); - let svc = service_fn(move |req: Request| { - let conn_state = conn_state.clone(); - async move { - let req = req.map(box_body_from_incoming); - Ok::<_, std::convert::Infallible>(handle(req, conn_state).await) - } - }); - - tokio::spawn(async move { - if let Err(err) = http1::Builder::new() - .preserve_header_case(true) - .title_case_headers(true) - .serve_connection(io, svc) - .with_upgrades() - .await - { - tracing::warn!("docker gateway connection failed: {err}"); - } - }); - } - } - } -} - async fn handle(mut req: Request, conn: Arc) -> Response { - let id = match conn.identity.clone() { - Some(identity) => identity, - None => { - return docker_error( - StatusCode::UNAUTHORIZED, - format!("unauthorized peer {}", conn.peer), - ); - } - }; - + let id = conn.identity.clone(); let (ver_opt, segs) = split_version_and_segments(req.uri().path()); let version_prefix = ver_opt.unwrap_or_else(|| "".to_string()); @@ -919,32 +715,6 @@ fn build_label_filter_query(required_labels: &[String], include_all: bool) -> St encode_query_pairs(&query_pairs) } -async fn shutdown_signal() -> ShutdownReason { - #[cfg(unix)] - { - use tokio::signal::unix::{SignalKind, signal}; - - match signal(SignalKind::terminate()) { - Ok(mut sigterm) => { - tokio::select! { - _ = tokio::signal::ctrl_c() => ShutdownReason::Interrupt, - _ = sigterm.recv() => ShutdownReason::Terminated, - } - } - Err(_) => { - let _ = tokio::signal::ctrl_c().await; - ShutdownReason::Interrupt - } - } - } - - #[cfg(not(unix))] - { - let _ = tokio::signal::ctrl_c().await; - return ShutdownReason::Interrupt; - } -} - async fn forward(mut req: Request, state: Arc) -> Response { if let Err(resp) = rewrite_to_docker_uri(&mut req, &state.cfg.docker_sock) { return *resp; diff --git a/runtime/docker-gateway/src/main.rs b/runtime/docker-gateway/src/main.rs deleted file mode 100644 index a6f0f670..00000000 --- a/runtime/docker-gateway/src/main.rs +++ /dev/null @@ -1,29 +0,0 @@ -use amber_docker_gateway::{DockerGatewayConfig, run}; -use tracing_subscriber::EnvFilter; - -#[tokio::main] -async fn main() { - init_tracing(); - - let config = match DockerGatewayConfig::from_env() { - Ok(config) => config, - Err(err) => { - tracing::error!("docker gateway config error: {err}"); - std::process::exit(1); - } - }; - - if let Err(err) = run(config).await { - tracing::error!("docker gateway failed: {err}"); - std::process::exit(1); - } -} - -fn init_tracing() { - let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("warn")); - tracing_subscriber::fmt() - .with_env_filter(filter) - .with_target(false) - .without_time() - .init(); -} diff --git a/runtime/docker-gateway/src/tests/compose.rs b/runtime/docker-gateway/src/tests/compose.rs index e5b8cf6c..437594e3 100644 --- a/runtime/docker-gateway/src/tests/compose.rs +++ b/runtime/docker-gateway/src/tests/compose.rs @@ -2,7 +2,7 @@ use super::*; #[tokio::test(flavor = "multi_thread")] async fn compose_in_compose_setup_injects_scoped_labels() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::POST, "/networks/create", @@ -167,7 +167,7 @@ async fn compose_in_compose_setup_injects_scoped_labels() { #[tokio::test(flavor = "multi_thread")] async fn compose_in_compose_teardown_allows_owned_resource_removal() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/containers/inner-workload/json", @@ -289,7 +289,7 @@ async fn compose_in_compose_teardown_allows_owned_resource_removal() { #[tokio::test(flavor = "multi_thread")] async fn compose_in_compose_remove_orphans_rewrites_project_filters() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/containers/json", @@ -396,10 +396,8 @@ async fn compose_in_compose_remove_orphans_rewrites_project_filters() { async fn shutdown_cleanup_removes_owned_resources() { let docker = MockDocker::start().await; let state = State::new(DockerGatewayConfig { - listen: "127.0.0.1:23750".parse().expect("valid listen addr"), docker_sock: docker.socket_path.clone(), compose_project: TEST_PROJECT.to_string(), - callers: vec![default_caller()], }); docker.enqueue_json( @@ -503,7 +501,7 @@ async fn shutdown_cleanup_removes_owned_resources() { #[tokio::test(flavor = "multi_thread")] async fn versioned_paths_preserve_version_for_authorization_and_forwarding() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/v1.41/containers/workload/json", @@ -534,7 +532,7 @@ async fn versioned_paths_preserve_version_for_authorization_and_forwarding() { #[tokio::test(flavor = "multi_thread")] async fn exec_start_uses_cached_exec_to_container_mapping() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/containers/workload/json", @@ -590,7 +588,7 @@ async fn exec_start_uses_cached_exec_to_container_mapping() { #[tokio::test(flavor = "multi_thread")] async fn exec_start_falls_back_to_exec_inspect_when_cache_is_missing() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/exec/exec-2/json", @@ -629,7 +627,7 @@ async fn exec_start_falls_back_to_exec_inspect_when_cache_is_missing() { #[tokio::test(flavor = "multi_thread")] async fn exec_start_denies_access_when_exec_belongs_to_foreign_container() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/exec/exec-9/json", diff --git a/runtime/docker-gateway/src/tests/e2e.rs b/runtime/docker-gateway/src/tests/e2e.rs index 7200613f..5da0c2b9 100644 --- a/runtime/docker-gateway/src/tests/e2e.rs +++ b/runtime/docker-gateway/src/tests/e2e.rs @@ -36,25 +36,12 @@ async fn docker_daemon_e2e_enforces_scoping_and_policy() { let owned_volume = format!("amber-gw-owned-vol-{suffix}"); let foreign_volume = format!("amber-gw-foreign-vol-{suffix}"); - let listen = reserve_loopback_socket_addr(); - let config = DockerGatewayConfig { - listen, - docker_sock: docker_sock.clone(), - compose_project: compose_project.clone(), - callers: vec![CallerConfig { - host: "127.0.0.1".to_string(), - port: None, - component: component.clone(), - compose_service: component.clone(), - }], - }; - - let gateway_task = tokio::spawn(async move { - if let Err(err) = run(config).await { - panic!("gateway run failed in e2e test: {err}"); - } - }); - wait_until_gateway_listens(listen, &gateway_task).await; + let (listen, gateway_task) = start_gateway_listener( + docker_sock.clone(), + compose_project.clone(), + component.clone(), + ) + .await; send_docker_request( &docker_sock, @@ -358,39 +345,21 @@ async fn docker_daemon_e2e_enforces_multicaller_container_exec_and_upgrade() { let owned_network_b = format!("amber-gw-owned-net-b-{suffix}"); let owned_container_a = format!("amber-gw-owned-container-a-{suffix}"); - let (caller_a_socket, caller_a_port) = reserve_bound_loopback_socket(); - let (caller_b_socket, caller_b_port) = reserve_bound_loopback_socket(); - - let listen = reserve_loopback_socket_addr(); - let config = DockerGatewayConfig { - listen, - docker_sock: docker_sock.clone(), - compose_project: compose_project.clone(), - callers: vec![ - CallerConfig { - host: "127.0.0.1".to_string(), - port: Some(caller_a_port), - component: component_a.clone(), - compose_service: component_a.clone(), - }, - CallerConfig { - host: "127.0.0.1".to_string(), - port: Some(caller_b_port), - component: component_b.clone(), - compose_service: component_b.clone(), - }, - ], - }; - - let gateway_task = tokio::spawn(async move { - if let Err(err) = run(config).await { - panic!("gateway run failed in multicaller e2e test: {err}"); - } - }); - wait_until_gateway_listens(listen, &gateway_task).await; + let (listen_a, gateway_task_a) = start_gateway_listener( + docker_sock.clone(), + compose_project.clone(), + component_a.clone(), + ) + .await; + let (listen_b, gateway_task_b) = start_gateway_listener( + docker_sock.clone(), + compose_project.clone(), + component_b.clone(), + ) + .await; - let mut caller_a = GatewayClient::connect_from_socket(caller_a_socket, listen).await; - let mut caller_b = GatewayClient::connect_from_socket(caller_b_socket, listen).await; + let mut caller_a = GatewayClient::connect(listen_a).await; + let mut caller_b = GatewayClient::connect(listen_b).await; let create_network_a = caller_a .request( @@ -584,7 +553,8 @@ async fn docker_daemon_e2e_enforces_multicaller_container_exec_and_upgrade() { String::from_utf8_lossy(&allowed_exec_start.body) ); - gateway_task.abort(); + gateway_task_a.abort(); + gateway_task_b.abort(); docker_delete_container_best_effort(&docker_sock, &owned_container_a).await; docker_delete_network_best_effort(&docker_sock, &owned_network_a).await; docker_delete_network_best_effort(&docker_sock, &owned_network_b).await; @@ -626,25 +596,12 @@ async fn docker_daemon_e2e_container_create_without_network_mode() { let component = format!("amber-gw-e2e-netmode-comp-{suffix}"); let container_name = format!("amber-gw-e2e-netmode-ctr-{suffix}"); - let listen = reserve_loopback_socket_addr(); - let config = DockerGatewayConfig { - listen, - docker_sock: docker_sock.clone(), - compose_project: compose_project.clone(), - callers: vec![CallerConfig { - host: "127.0.0.1".to_string(), - port: None, - component: component.clone(), - compose_service: component.clone(), - }], - }; - - let gateway_task = tokio::spawn(async move { - if let Err(err) = run(config).await { - panic!("gateway run failed in network-mode e2e test: {err}"); - } - }); - wait_until_gateway_listens(listen, &gateway_task).await; + let (listen, gateway_task) = start_gateway_listener( + docker_sock.clone(), + compose_project.clone(), + component.clone(), + ) + .await; // Docker CLI v27+ sends EndpointsConfig:{"default":{}} when no // --network flag is given. The gateway should skip builtin names. diff --git a/runtime/docker-gateway/src/tests/gateway.rs b/runtime/docker-gateway/src/tests/gateway.rs index b5677e18..4164b7e1 100644 --- a/runtime/docker-gateway/src/tests/gateway.rs +++ b/runtime/docker-gateway/src/tests/gateway.rs @@ -1,145 +1,45 @@ use super::*; #[tokio::test(flavor = "multi_thread")] -async fn resolve_identity_uses_port_and_denies_ambiguous_component_matches() { - let state = State::new(DockerGatewayConfig { - listen: "127.0.0.1:23750".parse().expect("valid listen addr"), - docker_sock: PathBuf::from("/tmp/docker.sock"), - compose_project: TEST_PROJECT.to_string(), - callers: vec![default_caller()], - }); - - { - let mut callers = state.callers_by_ip.write().await; - callers.insert( - IpAddr::V4(Ipv4Addr::LOCALHOST), - vec![ - ResolvedCaller { - component: "component-a".to_string(), - port: Some(30001), - }, - ResolvedCaller { - component: "component-b".to_string(), - port: Some(30002), - }, - ], - ); - } - - let a = state - .resolve_identity("127.0.0.1:30001".parse().expect("valid addr")) - .await - .expect("caller a should match"); - assert_eq!(a.component, "component-a"); - - let b = state - .resolve_identity("127.0.0.1:30002".parse().expect("valid addr")) - .await - .expect("caller b should match"); - assert_eq!(b.component, "component-b"); - - let unmatched = state - .resolve_identity("127.0.0.1:30003".parse().expect("valid addr")) - .await; - assert!( - unmatched.is_none(), - "unexpected identity for unmatched port" - ); - - { - let mut callers = state.callers_by_ip.write().await; - callers.insert( - IpAddr::V4(Ipv4Addr::LOCALHOST), - vec![ - ResolvedCaller { - component: "component-a".to_string(), - port: None, - }, - ResolvedCaller { - component: "component-b".to_string(), - port: None, - }, - ], - ); - } - - let ambiguous = state - .resolve_identity("127.0.0.1:30010".parse().expect("valid addr")) - .await; - assert!( - ambiguous.is_none(), - "ambiguous peer mapping should fail closed" - ); -} - -#[tokio::test(flavor = "multi_thread")] -async fn rejects_unauthorized_peer_before_proxying() { - let gateway = GatewayHarness::start(vec![CallerConfig { - host: "127.0.0.2".to_string(), - port: None, - component: TEST_COMPONENT.to_string(), - compose_service: TEST_COMPONENT.to_string(), - }]) - .await; - - let response = send_gateway_request(gateway.addr, Method::GET, "/_ping", &[], &[]).await; - assert_eq!(response.status, StatusCode::UNAUTHORIZED); - assert!(response_message(&response.body).contains("unauthorized peer")); - assert!(gateway.requests().is_empty()); -} - -#[tokio::test(flavor = "multi_thread")] -async fn enforces_port_restrictions_for_authenticated_callers() { - let (allowed_socket, allowed_port) = reserve_bound_loopback_socket(); - let (denied_socket, denied_port) = reserve_bound_loopback_socket(); - - let gateway = GatewayHarness::start(vec![CallerConfig { - host: "127.0.0.1".to_string(), - port: Some(allowed_port), - component: TEST_COMPONENT.to_string(), - compose_service: TEST_COMPONENT.to_string(), - }]) - .await; +async fn mesh_peer_identity_labels_created_resources() { + let gateway = GatewayHarness::start("/component-b").await; gateway.enqueue_json( - Method::GET, - "/version", - StatusCode::OK, - serde_json::json!({"Version":"24.0"}), + Method::POST, + "/containers/create", + StatusCode::CREATED, + serde_json::json!({"Id":"new-container"}), ); - let denied = send_gateway_request_from_socket( - denied_socket, - denied_port, - gateway.addr, - Method::GET, - "/version", - &[], - &[], - ) - .await; - assert_eq!(denied.status, StatusCode::UNAUTHORIZED); - - let allowed = send_gateway_request_from_socket( - allowed_socket, - allowed_port, + let response = send_gateway_request( gateway.addr, - Method::GET, - "/version", - &[], - &[], + Method::POST, + "/containers/create", + &[("content-type", "application/json")], + br#"{"Image":"busybox"}"#, ) .await; - assert_eq!(allowed.status, StatusCode::OK); + assert_eq!(response.status, StatusCode::CREATED); let requests = gateway.requests(); - assert_eq!(requests.len(), 1); - assert_eq!(requests[0].method, Method::GET); - assert_eq!(requests[0].path, "/version"); + let create_req = requests + .iter() + .find(|req| req.method == Method::POST && req.path == "/containers/create") + .expect("create should be forwarded"); + let forwarded: serde_json::Value = + serde_json::from_slice(&create_req.body).expect("forwarded body json"); + let labels = forwarded + .get("Labels") + .and_then(|value| value.as_object()) + .expect("forwarded labels object"); + assert_eq!( + labels.get(AMBER_COMPONENT_LABEL).and_then(|v| v.as_str()), + Some("/component-b") + ); } #[tokio::test(flavor = "multi_thread")] async fn default_deny_blocks_unapproved_endpoints() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; let response = send_gateway_request(gateway.addr, Method::GET, "/images/json", &[], &[]).await; assert_eq!(response.status, StatusCode::FORBIDDEN); @@ -149,7 +49,7 @@ async fn default_deny_blocks_unapproved_endpoints() { #[tokio::test(flavor = "multi_thread")] async fn allows_image_inspect_endpoint_for_prebuilt_workflows() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/images/busybox:latest/json", @@ -175,7 +75,7 @@ async fn allows_image_inspect_endpoint_for_prebuilt_workflows() { #[tokio::test(flavor = "multi_thread")] async fn allows_image_inspect_endpoint_for_registry_qualified_names() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/images/ghcr.io/org/app:latest/json", @@ -201,7 +101,7 @@ async fn allows_image_inspect_endpoint_for_registry_qualified_names() { #[tokio::test(flavor = "multi_thread")] async fn allows_image_pull_with_from_image() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::POST, "/images/create", @@ -231,7 +131,7 @@ async fn allows_image_pull_with_from_image() { #[tokio::test(flavor = "multi_thread")] async fn allows_image_pull_without_explicit_tag() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::POST, "/images/create", @@ -260,7 +160,7 @@ async fn allows_image_pull_without_explicit_tag() { #[tokio::test(flavor = "multi_thread")] async fn allows_image_pull_with_registry_qualified_name() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::POST, "/images/create", @@ -288,7 +188,7 @@ async fn allows_image_pull_with_registry_qualified_name() { #[tokio::test(flavor = "multi_thread")] async fn blocks_image_import_with_from_src() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; let response = send_gateway_request( gateway.addr, @@ -305,7 +205,7 @@ async fn blocks_image_import_with_from_src() { #[tokio::test(flavor = "multi_thread")] async fn blocks_image_create_with_no_query_params() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; let response = send_gateway_request(gateway.addr, Method::POST, "/images/create", &[], &[]).await; @@ -316,7 +216,7 @@ async fn blocks_image_create_with_no_query_params() { #[tokio::test(flavor = "multi_thread")] async fn blocks_image_create_with_both_from_image_and_from_src() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; let response = send_gateway_request( gateway.addr, @@ -332,7 +232,7 @@ async fn blocks_image_create_with_both_from_image_and_from_src() { #[tokio::test(flavor = "multi_thread")] async fn allows_image_pull_with_version_prefix() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::POST, "/v1.45/images/create", @@ -357,7 +257,7 @@ async fn allows_image_pull_with_version_prefix() { #[tokio::test(flavor = "multi_thread")] async fn allows_info_endpoint() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/info", @@ -376,7 +276,7 @@ async fn allows_info_endpoint() { #[tokio::test(flavor = "multi_thread")] async fn blocks_network_connect_and_disconnect() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; let connect = send_gateway_request( gateway.addr, @@ -403,7 +303,7 @@ async fn blocks_network_connect_and_disconnect() { #[tokio::test(flavor = "multi_thread")] async fn create_injects_owner_labels_and_normalizes_compose_project() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::POST, "/containers/create", @@ -477,7 +377,7 @@ async fn create_allows_endpoints_config_with_builtin_default_network() { // Docker CLI v27+ sends EndpointsConfig:{"default":{}} when no // --network flag is given. Builtin names should be skipped, just // like add_network_mode_reference does for NetworkMode. - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::POST, "/containers/create", @@ -518,7 +418,7 @@ async fn create_allows_endpoints_config_with_builtin_default_network() { #[tokio::test(flavor = "multi_thread")] async fn create_rejects_foreign_network_in_endpoints_config() { // User-defined networks in EndpointsConfig must still be authorized. - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/networks/custom-net", @@ -552,7 +452,7 @@ async fn create_rejects_foreign_network_in_endpoints_config() { #[tokio::test(flavor = "multi_thread")] async fn create_rejects_foreign_network_reference() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/networks/shared-net", @@ -586,7 +486,7 @@ async fn create_rejects_foreign_network_reference() { #[tokio::test(flavor = "multi_thread")] async fn create_rejects_foreign_named_volume_bind() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/volumes/shared-volume", @@ -620,7 +520,7 @@ async fn create_rejects_foreign_named_volume_bind() { #[tokio::test(flavor = "multi_thread")] async fn create_allows_owned_resource_references() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/containers/base/json", @@ -690,7 +590,7 @@ async fn create_allows_owned_resource_references() { #[tokio::test(flavor = "multi_thread")] async fn list_and_prune_endpoints_force_required_label_filters() { - let gateway = GatewayHarness::start(vec![default_caller()]).await; + let gateway = GatewayHarness::start(TEST_COMPONENT).await; gateway.enqueue_json( Method::GET, "/containers/json", diff --git a/runtime/docker-gateway/src/tests/mod.rs b/runtime/docker-gateway/src/tests/mod.rs index 236a44c3..17de1275 100644 --- a/runtime/docker-gateway/src/tests/mod.rs +++ b/runtime/docker-gateway/src/tests/mod.rs @@ -1,6 +1,7 @@ use std::{ collections::VecDeque, - net::{IpAddr, Ipv4Addr, SocketAddrV4}, + env, + net::{Ipv4Addr, SocketAddr}, path::PathBuf, sync::{Arc, Mutex}, time::{Duration, SystemTime, UNIX_EPOCH}, @@ -11,7 +12,7 @@ use hyper::client::conn::http1 as client_http1; use tempfile::TempDir; use tokio::{ io::AsyncReadExt, - net::{TcpSocket, TcpStream, UnixListener}, + net::{TcpListener, TcpStream, UnixListener}, task::JoinHandle, time::timeout, }; @@ -234,23 +235,14 @@ struct GatewayHarness { } impl GatewayHarness { - async fn start(callers: Vec) -> Self { + async fn start(peer_id: impl Into) -> Self { let docker = MockDocker::start().await; - let addr = reserve_loopback_socket_addr(); - let config = DockerGatewayConfig { - listen: addr, - docker_sock: docker.socket_path.clone(), - compose_project: TEST_PROJECT.to_string(), - callers, - }; - - let task = tokio::spawn(async move { - if let Err(err) = run(config).await { - panic!("gateway run failed: {err}"); - } - }); - - wait_until_gateway_listens(addr, &task).await; + let (addr, task) = start_gateway_listener( + docker.socket_path.clone(), + TEST_PROJECT.to_string(), + peer_id.into(), + ) + .await; Self { addr, docker, task } } @@ -291,11 +283,10 @@ struct GatewayClient { } impl GatewayClient { - async fn connect_from_socket(socket: TcpSocket, addr: SocketAddr) -> Self { - let stream = socket - .connect(addr) + async fn connect(addr: SocketAddr) -> Self { + let stream = TcpStream::connect(addr) .await - .unwrap_or_else(|err| panic!("connect from bound local source socket failed: {err}")); + .unwrap_or_else(|err| panic!("connect to gateway failed: {err}")); let io = TokioIo::new(stream); let (sender, conn) = client_http1::handshake(io) .await @@ -398,35 +389,33 @@ impl Drop for GatewayClient { } } -fn reserve_bound_loopback_socket() -> (TcpSocket, u16) { - let socket = TcpSocket::new_v4().expect("create v4 tcp socket"); - socket - .bind(SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 0))) - .expect("bind loopback tcp socket"); - let local_port = socket.local_addr().expect("socket local addr").port(); - (socket, local_port) -} - -fn reserve_loopback_socket_addr() -> SocketAddr { - let listener = std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) - .expect("bind temporary loopback listener"); - let addr = listener.local_addr().expect("local addr"); - drop(listener); - addr -} - -async fn wait_until_gateway_listens(addr: SocketAddr, task: &JoinHandle<()>) { - for _ in 0..400 { - if TcpStream::connect(addr).await.is_ok() { - return; +async fn start_gateway_listener( + docker_sock: PathBuf, + compose_project: String, + peer_id: String, +) -> (SocketAddr, JoinHandle<()>) { + let runtime = DockerGatewayRuntime::new(DockerGatewayConfig { + docker_sock, + compose_project, + }) + .expect("valid gateway config"); + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .expect("bind gateway test listener"); + let addr = listener.local_addr().expect("gateway listener addr"); + let task = tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.expect("accept gateway test client"); + let runtime = runtime.clone(); + let peer_id = peer_id.clone(); + tokio::spawn(async move { + if let Err(err) = runtime.serve_connection(stream, peer_id).await { + eprintln!("gateway test connection failed: {err}"); + } + }); } - assert!( - !task.is_finished(), - "gateway task exited before listener was ready" - ); - tokio::task::yield_now().await; - } - panic!("gateway listener did not come up at {addr}"); + }); + (addr, task) } async fn send_gateway_request( @@ -442,22 +431,6 @@ async fn send_gateway_request( send_gateway_request_on_stream(stream, method, target, headers, body).await } -async fn send_gateway_request_from_socket( - socket: TcpSocket, - source_port: u16, - addr: SocketAddr, - method: Method, - target: &str, - headers: &[(&str, &str)], - body: &[u8], -) -> GatewayResponse { - let stream = socket - .connect(addr) - .await - .unwrap_or_else(|err| panic!("connect from local source port {source_port} failed: {err}")); - send_gateway_request_on_stream(stream, method, target, headers, body).await -} - async fn send_gateway_request_on_stream( stream: TcpStream, method: Method, @@ -501,15 +474,6 @@ async fn send_gateway_request_on_stream( GatewayResponse { status, body } } -fn default_caller() -> CallerConfig { - CallerConfig { - host: "127.0.0.1".to_string(), - port: None, - component: TEST_COMPONENT.to_string(), - compose_service: TEST_COMPONENT.to_string(), - } -} - fn container_labels(component: &str, project: &str) -> serde_json::Value { serde_json::json!({ "Config": { diff --git a/runtime/docker-gateway/src/tests/unit.rs b/runtime/docker-gateway/src/tests/unit.rs index a5075ff7..9ec4bba5 100644 --- a/runtime/docker-gateway/src/tests/unit.rs +++ b/runtime/docker-gateway/src/tests/unit.rs @@ -193,44 +193,16 @@ fn parse_container_create_references_extracts_resources() { } #[test] -fn validate_rejects_empty_caller_host() { +fn validate_rejects_empty_compose_project() { let config = DockerGatewayConfig { - listen: "127.0.0.1:23750".parse().expect("valid listen addr"), docker_sock: PathBuf::from("/tmp/docker.sock"), - compose_project: TEST_PROJECT.to_string(), - callers: vec![CallerConfig { - host: " ".to_string(), - port: None, - component: TEST_COMPONENT.to_string(), - compose_service: TEST_COMPONENT.to_string(), - }], + compose_project: " ".to_string(), }; - let err = DockerGatewayConfig::validate(config).expect_err("empty host should fail"); - assert!( - err.to_string().contains("caller host must not be empty"), - "{err}" - ); -} - -#[test] -fn validate_rejects_empty_caller_component() { - let config = DockerGatewayConfig { - listen: "127.0.0.1:23750".parse().expect("valid listen addr"), - docker_sock: PathBuf::from("/tmp/docker.sock"), - compose_project: TEST_PROJECT.to_string(), - callers: vec![CallerConfig { - host: "localhost".to_string(), - port: None, - component: " ".to_string(), - compose_service: TEST_COMPONENT.to_string(), - }], - }; - - let err = DockerGatewayConfig::validate(config).expect_err("empty component should fail"); + let err = DockerGatewayConfig::validate(config).expect_err("empty compose project should fail"); assert!( err.to_string() - .contains("caller component must not be empty"), + .contains("compose_project must not be empty"), "{err}" ); } diff --git a/runtime/helper/src/lib.rs b/runtime/helper/src/lib.rs index 716f5b9b..60bd7c3e 100644 --- a/runtime/helper/src/lib.rs +++ b/runtime/helper/src/lib.rs @@ -35,6 +35,7 @@ const IP_BIN: &str = "/usr/sbin/ip"; const IPTABLES_BIN: &str = "/usr/sbin/iptables"; const IP6TABLES_BIN: &str = "/usr/sbin/ip6tables"; const DEFAULT_EGRESS_CHAIN: &str = "AMBER_EGRESS"; +const DEFAULT_INGRESS_CHAIN: &str = "AMBER_INGRESS"; const HOST_GATEWAY_NAME: &str = "host.docker.internal"; const BLOCKED_IPV4_CIDRS: &[&str] = &[ "10.0.0.0/8", @@ -96,6 +97,11 @@ pub enum HelperError { pub type Result = std::result::Result; +pub fn install_default_network_guard(allowed_ingress_tcp_ports: &[u16]) -> Result<()> { + install_default_ingress_guard(allowed_ingress_tcp_ports)?; + install_default_egress_guard() +} + pub fn install_default_egress_guard() -> Result<()> { let host_gateway_ips = resolve_host_gateway_ips(); install_egress_guard_family(IPTABLES_BIN, "-4", BLOCKED_IPV4_CIDRS, &host_gateway_ips)?; @@ -103,6 +109,12 @@ pub fn install_default_egress_guard() -> Result<()> { Ok(()) } +fn install_default_ingress_guard(allowed_tcp_ports: &[u16]) -> Result<()> { + install_ingress_guard_family(IPTABLES_BIN, allowed_tcp_ports)?; + install_ingress_guard_family(IP6TABLES_BIN, allowed_tcp_ports)?; + Ok(()) +} + pub fn wait_for_mesh_config_scope( config_path: &Path, expected_scope: &str, @@ -221,6 +233,71 @@ fn install_egress_guard_family( Ok(()) } +fn install_ingress_guard_family(iptables_bin: &str, allowed_tcp_ports: &[u16]) -> Result<()> { + run_command_allow_failure( + iptables_bin, + ["-w", "-D", "INPUT", "-j", DEFAULT_INGRESS_CHAIN], + )?; + run_command_allow_failure(iptables_bin, ["-w", "-F", DEFAULT_INGRESS_CHAIN])?; + run_command_allow_failure(iptables_bin, ["-w", "-X", DEFAULT_INGRESS_CHAIN])?; + + run_command(iptables_bin, ["-w", "-N", DEFAULT_INGRESS_CHAIN])?; + run_command( + iptables_bin, + ["-w", "-A", "INPUT", "-j", DEFAULT_INGRESS_CHAIN], + )?; + run_command( + iptables_bin, + [ + "-w", + "-A", + DEFAULT_INGRESS_CHAIN, + "-m", + "conntrack", + "--ctstate", + "ESTABLISHED,RELATED", + "-j", + "RETURN", + ], + )?; + run_command( + iptables_bin, + [ + "-w", + "-A", + DEFAULT_INGRESS_CHAIN, + "-i", + "lo", + "-j", + "RETURN", + ], + )?; + + for port in allowed_tcp_ports { + let port = port.to_string(); + run_command( + iptables_bin, + [ + "-w", + "-A", + DEFAULT_INGRESS_CHAIN, + "-p", + "tcp", + "--dport", + port.as_str(), + "-j", + "RETURN", + ], + )?; + } + + run_command( + iptables_bin, + ["-w", "-A", DEFAULT_INGRESS_CHAIN, "-j", "REJECT"], + )?; + Ok(()) +} + fn read_route_snapshot(family_flag: &str) -> Result { let output = run_command( IP_BIN, diff --git a/runtime/helper/src/main.rs b/runtime/helper/src/main.rs index 0ef1e7e8..c2f42932 100644 --- a/runtime/helper/src/main.rs +++ b/runtime/helper/src/main.rs @@ -15,7 +15,8 @@ use std::{ }; use amber_helper::{ - HelperError, RunPlan, build_run_plan, install_default_egress_guard, wait_for_mesh_config_scope, + HelperError, RunPlan, build_run_plan, install_default_egress_guard, + install_default_network_guard, wait_for_mesh_config_scope, }; use amber_mesh::telemetry::{ COMPONENT_MONIKER_ENV, OtlpIdentity, OtlpInstallMode, SCENARIO_SCOPE_ENV, SubscriberFormat, @@ -148,6 +149,20 @@ fn run_main() -> Result { install_default_egress_guard()?; Ok(ExitCode::SUCCESS) } + "install-network-guard" => { + let allowed_ingress_tcp_ports = args + .map(|value| { + value.parse::().map_err(|err| { + HelperError::Msg(format!( + "install-network-guard allowed ingress port must be a TCP port \ + number: {err}" + )) + }) + }) + .collect::, _>>()?; + install_default_network_guard(&allowed_ingress_tcp_ports)?; + Ok(ExitCode::SUCCESS) + } _ => Err(usage_error()), } } @@ -155,7 +170,7 @@ fn run_main() -> Result { fn usage_error() -> HelperError { HelperError::Msg( "usage: amber-helper " + [TIMEOUT_SECONDS]|install-default-egress-guard|install-network-guard [TCP_PORT...]>" .to_string(), ) } diff --git a/runtime/mesh/src/lib.rs b/runtime/mesh/src/lib.rs index e0ed924d..27fcb39f 100644 --- a/runtime/mesh/src/lib.rs +++ b/runtime/mesh/src/lib.rs @@ -328,6 +328,10 @@ pub enum InboundTarget { Local { port: u16, }, + DockerGateway { + docker_sock: PathBuf, + compose_project_env: String, + }, External { url_env: String, #[serde(default, skip_serializing_if = "is_false")] diff --git a/runtime/router/Cargo.toml b/runtime/router/Cargo.toml index 122ed2eb..afab6a6c 100644 --- a/runtime/router/Cargo.toml +++ b/runtime/router/Cargo.toml @@ -3,6 +3,7 @@ name = "amber-router" edition = "2024" [dependencies] +amber-docker-gateway = { workspace = true } amber-mesh = { workspace = true } aho-corasick = "1.1.3" base64 = { workspace = true } diff --git a/runtime/router/src/control.rs b/runtime/router/src/control.rs index d68c667d..28c0012f 100644 --- a/runtime/router/src/control.rs +++ b/runtime/router/src/control.rs @@ -24,6 +24,7 @@ pub(super) struct InboundRuntime { client: Arc, a2a_url_rewrite_table: Arc, dynamic_caps: Option>, + docker_gateways: DockerGatewayRuntimes, } #[derive(Debug, Deserialize)] @@ -158,6 +159,7 @@ pub async fn run_with_listeners( let trust = Arc::new(TrustBundle::new(&config)?); let inbound_routes = Arc::new(build_inbound_routes(&config)?); validate_outbound_routes(&config)?; + let docker_gateways = Arc::new(build_docker_gateway_runtimes(&config)?); let a2a_url_rewrite_table = Arc::new(a2a::UrlRewriteTable::from_routes( &config.inbound, &config.outbound, @@ -193,6 +195,7 @@ pub async fn run_with_listeners( client: client.clone(), a2a_url_rewrite_table: a2a_url_rewrite_table.clone(), dynamic_caps: dynamic_caps.clone(), + docker_gateways: docker_gateways.clone(), }, listeners_by_route.mesh.take(), )); @@ -379,6 +382,7 @@ pub(super) async fn handle_inbound( client, a2a_url_rewrite_table, dynamic_caps, + docker_gateways, } = state; let noise_keys = noise_keys_for_identity(&config.identity)?; let mut session = accept_noise(stream, &noise_keys, &trust).await?; @@ -425,6 +429,24 @@ pub(super) async fn handle_inbound( proxy_noise_to_plain(&mut session, target).await?; } } + InboundTarget::DockerGateway { .. } => { + if route.protocol != MeshProtocol::Tcp { + return Err(RouterError::InvalidConfig(format!( + "docker gateway route {} must use tcp protocol", + route.route_id + ))); + } + let gateway = docker_gateways + .get(&route.route_id) + .cloned() + .ok_or_else(|| { + RouterError::InvalidConfig(format!( + "missing docker gateway runtime for route {}", + route.route_id + )) + })?; + proxy_noise_to_docker_gateway(&mut session, remote_id, gateway).await?; + } InboundTarget::External { ref url_env, optional, @@ -1477,6 +1499,15 @@ pub(super) fn control_protocol(value: &str) -> Result { pub(super) fn build_inbound_routes(config: &MeshConfig) -> Result { let mut map = HashMap::new(); for route in &config.inbound { + if matches!(route.target, InboundTarget::DockerGateway { .. }) + && route.protocol != MeshProtocol::Tcp + { + return Err(RouterError::InvalidConfig(format!( + "inbound route {} targets the docker gateway but uses {} protocol", + route.route_id, + protocol_string(route.protocol) + ))); + } if route.protocol != MeshProtocol::Http && !route.http_plugins.is_empty() { return Err(RouterError::InvalidConfig(format!( "inbound route {} has http plugins but uses {} protocol", @@ -1500,6 +1531,38 @@ pub(super) fn build_inbound_routes(config: &MeshConfig) -> Result Result>, RouterError> { + let mut runtimes = HashMap::new(); + for route in &config.inbound { + let InboundTarget::DockerGateway { + docker_sock, + compose_project_env, + } = &route.target + else { + continue; + }; + let compose_project = env::var(compose_project_env) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .ok_or_else(|| { + RouterError::InvalidConfig(format!( + "docker gateway route {} requires non-empty {compose_project_env}", + route.route_id + )) + })?; + let runtime = DockerGatewayRuntime::new(DockerGatewayConfig { + docker_sock: docker_sock.clone(), + compose_project, + }) + .map_err(|err| RouterError::InvalidConfig(err.to_string()))?; + runtimes.insert(route.route_id.clone(), Arc::new(runtime)); + } + Ok(runtimes) +} + pub(super) fn validate_outbound_routes(config: &MeshConfig) -> Result<(), RouterError> { for route in &config.outbound { if route.protocol != MeshProtocol::Http && !route.http_plugins.is_empty() { diff --git a/runtime/router/src/lib.rs b/runtime/router/src/lib.rs index 592b52dd..53e0b181 100644 --- a/runtime/router/src/lib.rs +++ b/runtime/router/src/lib.rs @@ -9,6 +9,7 @@ use std::{ task::{Context, Poll}, }; +use amber_docker_gateway::{DockerGatewayConfig, DockerGatewayRuntime}; use amber_mesh::{ HttpRoutePlugin, InboundRoute, InboundTarget, MeshConfig, MeshConfigPublic, MeshIdentity, MeshIdentityPublic, MeshIdentitySecret, MeshPeer, MeshProtocol, OutboundRoute, @@ -208,6 +209,29 @@ impl HttpExchangeLabels { ) } } + InboundTarget::DockerGateway { .. } => { + if router_forwarded_export { + ( + HttpEdgeKind::Export, + false, + None, + source_from_open + .clone() + .unwrap_or_else(|| capability.clone()), + Some(local_id.clone()), + capability.clone(), + ) + } else { + ( + HttpEdgeKind::Binding, + true, + Some(remote_id.clone()), + source_from_open.unwrap_or_else(|| capability.clone()), + Some(local_id.clone()), + capability.clone(), + ) + } + } InboundTarget::External { .. } => ( HttpEdgeKind::ExternalSlot, true, @@ -314,6 +338,7 @@ struct OutboundHttpProxyState { } type HttpClient = Client>, BoxBody>; +type DockerGatewayRuntimes = Arc>>; #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum BodyMode { diff --git a/runtime/router/src/tests.rs b/runtime/router/src/tests.rs index 6fa61a26..5d0f6ab4 100644 --- a/runtime/router/src/tests.rs +++ b/runtime/router/src/tests.rs @@ -234,6 +234,35 @@ fn build_inbound_routes_rejects_http_plugins_on_non_local_target() { } } +#[test] +fn build_inbound_routes_rejects_non_tcp_docker_gateway_route() { + let config = MeshConfig { + inbound: vec![InboundRoute { + route_id: "docker-route".to_string(), + capability: "docker".to_string(), + capability_kind: Some("docker".to_string()), + capability_profile: None, + protocol: MeshProtocol::Http, + http_plugins: Vec::new(), + target: InboundTarget::DockerGateway { + docker_sock: "/var/run/docker.sock".into(), + compose_project_env: "COMPOSE_PROJECT_NAME".to_string(), + }, + allowed_issuers: vec!["peer-a".to_string()], + }], + ..test_mesh_config() + }; + + let err = build_inbound_routes(&config) + .expect_err("docker gateway routes must stay on the tcp mesh path"); + match err { + RouterError::InvalidConfig(message) => { + assert!(message.contains("targets the docker gateway but uses http protocol")); + } + other => panic!("unexpected error: {other}"), + } +} + #[test] fn validate_outbound_routes_rejects_http_plugins_on_non_http_route() { let config = MeshConfig { diff --git a/runtime/router/src/transport.rs b/runtime/router/src/transport.rs index 69fe05c4..1fcd8e6d 100644 --- a/runtime/router/src/transport.rs +++ b/runtime/router/src/transport.rs @@ -367,6 +367,24 @@ pub(super) async fn proxy_noise_to_external_tcp( proxy_noise_to_plain(session, upstream).await } +pub(super) async fn proxy_noise_to_docker_gateway( + session: &mut NoiseSession, + peer_id: String, + gateway: Arc, +) -> Result<(), RouterError> { + let (local, remote) = duplex(64 * 1024); + let mut noise_session = session.clone(); + let bridge = tokio::spawn(async move { proxy_noise_to_plain(&mut noise_session, local).await }); + + gateway + .serve_connection(remote, peer_id) + .await + .map_err(|err| RouterError::Transport(err.to_string()))?; + + let _ = bridge.await; + Ok(()) +} + #[allow(clippy::too_many_arguments)] pub(super) async fn proxy_noise_to_local_http( session: &mut NoiseSession, From a549e1abc6869bf45498b11ddf88a977e23b4558 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Sun, 10 May 2026 16:42:50 +0800 Subject: [PATCH 23/25] Fix site-controller state persistence --- cli/src/mixed_run/launch_bundle.rs | 4 +- cli/src/mixed_run/tests.rs | 8 +- cli/tests/mixed_run/dry_run.rs | 38 ++++ cli/tests/mixed_run/lifecycle.rs | 143 ++++++++++++++ cli/tests/mixed_run/support.rs | 9 +- .../src/default_runtime/compose_controller.rs | 77 +++++++- .../default_runtime/kubernetes_controller.rs | 183 ++++++++++++------ runtime/site-controller/src/tests.rs | 41 +++- 8 files changed, 418 insertions(+), 85 deletions(-) diff --git a/cli/src/mixed_run/launch_bundle.rs b/cli/src/mixed_run/launch_bundle.rs index 0c86b3ae..b1932831 100644 --- a/cli/src/mixed_run/launch_bundle.rs +++ b/cli/src/mixed_run/launch_bundle.rs @@ -145,12 +145,12 @@ pub(crate) fn prepare_site_state_root(site_state_root: &Path, kind: SiteKind) -> ) })? .permissions(); - permissions.set_mode(0o777); + permissions.set_mode(0o700); fs::set_permissions(site_state_root, permissions) .into_diagnostic() .wrap_err_with(|| { format!( - "failed to make compose site state directory writable {}", + "failed to restrict compose site state directory {}", site_state_root.display() ) })?; diff --git a/cli/src/mixed_run/tests.rs b/cli/src/mixed_run/tests.rs index ddbea877..30b72a59 100644 --- a/cli/src/mixed_run/tests.rs +++ b/cli/src/mixed_run/tests.rs @@ -38,7 +38,7 @@ fn site_state_paths_are_site_scoped() { #[cfg(unix)] #[test] -fn compose_site_state_root_is_writable_by_container_uid_mappings() { +fn compose_site_state_root_is_private_to_the_launching_user() { let temp = TempDir::new().expect("temp dir should create"); let site_state_root = temp.path().join("state").join("compose-local"); @@ -51,9 +51,9 @@ fn compose_site_state_root_is_writable_by_container_uid_mappings() { .mode() & 0o777; assert_eq!( - mode, 0o777, - "compose site controllers run in Docker and must be able to persist state through Linux \ - bind mounts even when the daemon remaps container uid 0" + mode, 0o700, + "compose site controller state contains control-plane data and must not be writable by \ + unrelated local users" ); } diff --git a/cli/tests/mixed_run/dry_run.rs b/cli/tests/mixed_run/dry_run.rs index 63d6edbb..20abb90c 100644 --- a/cli/tests/mixed_run/dry_run.rs +++ b/cli/tests/mixed_run/dry_run.rs @@ -1,4 +1,6 @@ use std::collections::BTreeSet; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt as _; use amber_compiler::run_plan::FrameworkComponentControllerMoniker; @@ -177,6 +179,42 @@ fn mixed_run_dry_run_emits_launch_bundle_without_starting_sites() { .is_empty(), "dry-run should record compose launch commands" ); + #[cfg(unix)] + { + let compose_state_mode = fs::metadata(bundle_root.join("state").join("compose_local")) + .expect("compose site state root should exist") + .permissions() + .mode() + & 0o777; + assert_eq!( + compose_state_mode, 0o700, + "compose controller state should be private in the materialized launch bundle" + ); + + let compose_yaml_path = Path::new( + compose_site["artifact_dir"] + .as_str() + .expect("launch bundle should record compose artifact dir"), + ) + .join("compose.yaml"); + let compose_yaml: serde_yaml::Value = serde_yaml::from_str( + &fs::read_to_string(&compose_yaml_path).expect("compose yaml should read"), + ) + .expect("compose yaml should parse"); + let controller = compose_yaml["services"]["amber-site-controller"] + .as_mapping() + .expect("compose yaml should include the site controller service"); + let expected_user = format!("{}:{}", unsafe { libc::getuid() }, unsafe { + libc::getgid() + }); + assert_eq!( + controller + .get(serde_yaml::Value::String("user".to_string())) + .and_then(serde_yaml::Value::as_str), + Some(expected_user.as_str()), + "compose site controller should run as the launching user" + ); + } let stitching = launch_bundle["stitching"] .as_array() .expect("launch bundle stitching should serialize as an array"); diff --git a/cli/tests/mixed_run/lifecycle.rs b/cli/tests/mixed_run/lifecycle.rs index f7d4c760..3ec31f7b 100644 --- a/cli/tests/mixed_run/lifecycle.rs +++ b/cli/tests/mixed_run/lifecycle.rs @@ -665,3 +665,146 @@ fn mixed_run_recovers_when_kubernetes_site_is_temporarily_unreachable() { stop_proxy(&mut proxy); run.stop(); } + +#[test] +#[ignore = "requires docker + kind + kubectl; run manually or in CI"] +fn mixed_run_kubernetes_site_controller_state_survives_pod_recreation() { + let temp = temp_output_dir("mixed-run-kind-controller-pvc-"); + let kubeconfig = temp.path().join("kubeconfig"); + let kind_cluster = KindCluster::from_env_or_create(&kubeconfig); + ensure_kind_internal_images(&kind_cluster); + let kubeconfig_env = kind_cluster.kubeconfig.display().to_string(); + let fixture = write_single_site_kind_fixture(temp.path(), &kind_cluster); + let storage_root = temp.path().join("state"); + let mut run = run_manifest_with_env( + &fixture.manifest, + &fixture.placement, + &storage_root, + &[("KUBECONFIG", &kubeconfig_env)], + ); + + let kind_state = wait_for_state_status( + &run.run_root, + "kind_local", + "running", + Duration::from_secs(120), + ); + let namespace = kind_state["kubernetes_namespace"] + .as_str() + .expect("kubernetes site should publish namespace"); + let pod = kubernetes_site_controller_pod_name(&kind_state, namespace); + + let pvc_phase = kubectl_for_manager_state(&kind_state) + .arg("-n") + .arg(namespace) + .arg("get") + .arg("pvc") + .arg("amber-site-controller-state") + .arg("-o") + .arg("jsonpath={.status.phase}") + .output() + .expect("failed to query site-controller state PVC"); + assert!( + pvc_phase.status.success(), + "failed to query site-controller state PVC:\n{}", + String::from_utf8_lossy(&pvc_phase.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&pvc_phase.stdout).trim(), + "Bound", + "running kubernetes site-controller should have a bound state PVC" + ); + + let secret_status = kubectl_for_manager_state(&kind_state) + .arg("-n") + .arg(namespace) + .arg("get") + .arg("secret") + .arg("amber-site-controller-seed") + .status() + .expect("failed to query site-controller seed Secret"); + assert!( + secret_status.success(), + "kubernetes site-controller seed should be stored as a Secret" + ); + + let sentinel_path = "/amber/site/state/pvc-restart-sentinel"; + let write_status = kubectl_for_manager_state(&kind_state) + .arg("-n") + .arg(namespace) + .arg("exec") + .arg(&pod) + .arg("-c") + .arg(KUBERNETES_SITE_CONTROLLER_MAIN_CONTAINER) + .arg("--") + .arg("sh") + .arg("-lc") + .arg(format!("printf preserved > {sentinel_path}")) + .status() + .expect("failed to write state sentinel in site-controller pod"); + assert!(write_status.success(), "failed to write state sentinel"); + + let delete_status = kubectl_for_manager_state(&kind_state) + .arg("-n") + .arg(namespace) + .arg("delete") + .arg("pod") + .arg(&pod) + .arg("--wait=true") + .status() + .expect("failed to delete site-controller pod"); + assert!( + delete_status.success(), + "failed to delete site-controller pod" + ); + + let mut replacement_ready = false; + for _ in 0..240 { + let status = kubectl_for_manager_state(&kind_state) + .arg("-n") + .arg(namespace) + .arg("wait") + .arg("pod") + .arg("-l") + .arg("amber.io/component=amber-site-controller") + .arg("--for=condition=Ready") + .arg("--timeout=1s") + .status() + .expect("failed to wait for replacement site-controller pod"); + if status.success() { + replacement_ready = true; + break; + } + std::thread::sleep(Duration::from_millis(500)); + } + assert!( + replacement_ready, + "replacement site-controller pod did not become ready" + ); + + let replacement_pod = kubernetes_site_controller_pod_name(&kind_state, namespace); + assert_ne!( + replacement_pod, pod, + "deployment should recreate the site-controller pod after deletion" + ); + let sentinel = kubectl_for_manager_state(&kind_state) + .arg("-n") + .arg(namespace) + .arg("exec") + .arg(&replacement_pod) + .arg("-c") + .arg(KUBERNETES_SITE_CONTROLLER_MAIN_CONTAINER) + .arg("--") + .arg("cat") + .arg(sentinel_path) + .output() + .expect("failed to read state sentinel after pod recreation"); + assert!( + sentinel.status.success(), + "state sentinel missing after pod recreation:\n{}", + String::from_utf8_lossy(&sentinel.stderr) + ); + assert_eq!(String::from_utf8_lossy(&sentinel.stdout), "preserved"); + + run.stop(); +} diff --git a/cli/tests/mixed_run/support.rs b/cli/tests/mixed_run/support.rs index 701e9f64..b6619845 100644 --- a/cli/tests/mixed_run/support.rs +++ b/cli/tests/mixed_run/support.rs @@ -139,7 +139,7 @@ ThreadingHTTPServer(("0.0.0.0", PORT), Handler).serve_forever() const TEST_APP_SOURCE_IMAGE: &str = "python:3.13-alpine"; const TEST_APP_LOCAL_IMAGE_REPOSITORY: &str = "amber-mixed-run-test-app"; -const KUBERNETES_SITE_CONTROLLER_MAIN_CONTAINER: &str = "main"; +pub(crate) const KUBERNETES_SITE_CONTROLLER_MAIN_CONTAINER: &str = "main"; #[derive(Clone, Debug, PartialEq, Eq)] struct DockerImageMeta { @@ -1795,7 +1795,10 @@ fn materialize_kubernetes_control_state(manager_state: &Value, state_path: &Path }); } -fn kubernetes_site_controller_pod_name(manager_state: &Value, namespace: &str) -> String { +pub(crate) fn kubernetes_site_controller_pod_name( + manager_state: &Value, + namespace: &str, +) -> String { let mut command = kubectl_for_manager_state(manager_state); let output = command .arg("-n") @@ -1823,7 +1826,7 @@ fn kubernetes_site_controller_pod_name(manager_state: &Value, namespace: &str) - pod } -fn kubectl_for_manager_state(manager_state: &Value) -> Command { +pub(crate) fn kubectl_for_manager_state(manager_state: &Value) -> Command { let mut command = Command::new("kubectl"); if let Some(kubeconfig) = env::var_os("AMBER_TEST_KIND_KUBECONFIG") { command.env("KUBECONFIG", kubeconfig); diff --git a/runtime/site-controller/src/default_runtime/compose_controller.rs b/runtime/site-controller/src/default_runtime/compose_controller.rs index e43926ca..57aa16a1 100644 --- a/runtime/site-controller/src/default_runtime/compose_controller.rs +++ b/runtime/site-controller/src/default_runtime/compose_controller.rs @@ -1,3 +1,5 @@ +#[cfg(unix)] +use std::os::unix::fs::MetadataExt as _; use std::{fs, path::Path}; use serde_json::json; @@ -9,7 +11,7 @@ const COMPOSE_ROUTER_SERVICE_NAME: &str = "amber-router"; const COMPOSE_ROUTER_CONTROL_INIT_SERVICE_NAME: &str = "amber-router-control-init"; const COMPOSE_ROUTER_CONTROL_SOCKET_DIR: &str = "/amber/control"; const COMPOSE_ROUTER_CONTROL_VOLUME_NAME: &str = "amber-router-control"; -const COMPOSE_ROUTER_RUNTIME_GID: &str = "65532"; +const COMPOSE_ROUTER_RUNTIME_GID: u32 = 65532; const DOCKER_SOCK_PATH: &str = "/var/run/docker.sock"; const COMPOSE_CONTROLLER_PLAN_PATH: &str = "/amber/site/state/site-controller-plan.json"; @@ -30,6 +32,50 @@ fn uses_service_network_mode(service: &serde_yaml::Mapping) -> bool { .is_some_and(|mode| mode.starts_with("service:")) } +fn compose_controller_user() -> String { + #[cfg(unix)] + { + format!("{}:{}", unsafe { libc::getuid() }, unsafe { + libc::getgid() + }) + } + + #[cfg(not(unix))] + { + "0:0".to_string() + } +} + +#[cfg(unix)] +fn compose_controller_group_add() -> Vec { + compose_controller_group_add_for( + unsafe { libc::getgid() }, + fs::metadata(DOCKER_SOCK_PATH) + .ok() + .map(|metadata| metadata.gid()), + ) +} + +#[cfg(unix)] +fn compose_controller_group_add_for( + runtime_gid: u32, + docker_socket_gid: Option, +) -> Vec { + let mut groups = vec![COMPOSE_ROUTER_RUNTIME_GID]; + if let Some(docker_socket_gid) = docker_socket_gid + && docker_socket_gid != runtime_gid + && !groups.contains(&docker_socket_gid) + { + groups.push(docker_socket_gid); + } + groups.into_iter().map(|gid| gid.to_string()).collect() +} + +#[cfg(not(unix))] +fn compose_controller_group_add() -> Vec { + vec![COMPOSE_ROUTER_RUNTIME_GID.to_string()] +} + fn ensure_controller_environment( service: &mut serde_yaml::Mapping, plan: &SiteControllerPlan, @@ -174,10 +220,15 @@ pub fn inject_compose_site_controller_with_mount_sources( })?; service.insert(yaml_string("image"), yaml_string(controller_image)); - service.insert(yaml_string("user"), yaml_string("0:0")); + service.insert(yaml_string("user"), yaml_string(&compose_controller_user())); service.insert( yaml_string("group_add"), - serde_yaml::Value::Sequence(vec![yaml_string(COMPOSE_ROUTER_RUNTIME_GID)]), + serde_yaml::Value::Sequence( + compose_controller_group_add() + .into_iter() + .map(|gid| yaml_string(&gid)) + .collect(), + ), ); service.insert( yaml_string("healthcheck"), @@ -249,6 +300,26 @@ mod tests { use super::*; + #[cfg(unix)] + #[test] + fn compose_controller_groups_include_router_and_docker_socket_groups() { + assert_eq!( + compose_controller_group_add_for(1000, Some(1000)), + vec!["65532"], + "the controller already has its primary runtime group" + ); + assert_eq!( + compose_controller_group_add_for(1000, Some(998)), + vec!["65532", "998"], + "the controller should join the Docker socket group without becoming root" + ); + assert_eq!( + compose_controller_group_add_for(1000, Some(COMPOSE_ROUTER_RUNTIME_GID)), + vec!["65532"], + "router runtime group should not be duplicated" + ); + } + fn test_plan(run_root: &Path) -> SiteControllerPlan { SiteControllerPlan { schema: "amber.framework_component.site_controller_plan".to_string(), diff --git a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs index 17e51ede..8fde5782 100644 --- a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs +++ b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs @@ -17,20 +17,22 @@ const KUBERNETES_ROUTER_SERVICE_PATH: &str = "04-services/amber-router.yaml"; const KUBERNETES_ROUTER_NETPOL_PATH: &str = "05-networkpolicies/amber-router-netpol.yaml"; const KUBERNETES_CONTROLLER_NETPOL_PATH: &str = "05-networkpolicies/amber-site-controller-netpol.yaml"; -const KUBERNETES_CONTROLLER_SEED_CONFIGMAP_PATH: &str = - "01-configmaps/amber-site-controller-seed.yaml"; -const KUBERNETES_CONTROLLER_SEED_CONFIGMAP_PATH_PREFIX: &str = - "01-configmaps/amber-site-controller-seed"; -const KUBERNETES_CONTROLLER_SEED_CONFIGMAP_DATA_LIMIT: usize = 64 * 1024; +const KUBERNETES_CONTROLLER_SEED_SECRET_PATH: &str = "01-secrets/amber-site-controller-seed.yaml"; +const KUBERNETES_CONTROLLER_SEED_SECRET_PATH_PREFIX: &str = "01-secrets/amber-site-controller-seed"; +const KUBERNETES_CONTROLLER_SEED_SECRET_DATA_LIMIT: usize = 64 * 1024; const KUBERNETES_CONTROLLER_SERVICE_ACCOUNT_PATH: &str = "02-rbac/amber-site-controller-sa.yaml"; const KUBERNETES_CONTROLLER_ROLE_PATH: &str = "02-rbac/amber-site-controller-role.yaml"; const KUBERNETES_CONTROLLER_ROLE_BINDING_PATH: &str = "02-rbac/amber-site-controller-rolebinding.yaml"; const KUBERNETES_CONTROLLER_DEPLOYMENT_PATH: &str = "03-deployments/amber-site-controller.yaml"; +const KUBERNETES_CONTROLLER_STATE_PVC_PATH: &str = + "03-persistentvolumeclaims/amber-site-controller-state.yaml"; #[cfg(test)] const KUBERNETES_CONTROLLER_SERVICE_PATH: &str = "04-services/amber-site-controller.yaml"; const KUBERNETES_CONTROLLER_SEED_VOLUME: &str = "controller-seed"; const KUBERNETES_CONTROLLER_STATE_VOLUME: &str = "controller-state"; +const KUBERNETES_CONTROLLER_STATE_PVC_NAME: &str = "amber-site-controller-state"; +const KUBERNETES_CONTROLLER_STATE_STORAGE_REQUEST: &str = "1Gi"; const KUBERNETES_CONTROLLER_SITE_ROOT: &str = "/amber/site"; const KUBERNETES_CONTROLLER_STATE_ROOT: &str = "/amber/site/state"; const KUBERNETES_CONTROLLER_ARTIFACT_ROOT: &str = "/amber/site/artifact"; @@ -40,7 +42,7 @@ const KUBERNETES_CONTROLLER_DESIRED_LINKS_PATH: &str = "/amber/site/state/desire const KUBERNETES_CONTROLLER_SEED_ROOT: &str = "/amber/seed"; #[derive(Clone, Debug)] -struct KubernetesControllerSeedConfigMap { +struct KubernetesControllerSeedSecret { path: String, name: String, items: Vec, @@ -53,13 +55,32 @@ pub fn inject_kubernetes_site_controller( controller_image: &str, ) -> Result<()> { let embedded_plan = build_embedded_kubernetes_controller_plan(plan); - let seed_configmaps = - build_kubernetes_controller_seed_configmaps(artifact_root, plan, &embedded_plan)?; + let seed_secrets = + build_kubernetes_controller_seed_secrets(artifact_root, plan, &embedded_plan)?; let labels = kubernetes_controller_labels(); - for configmap in &seed_configmaps { - write_yaml_artifact(artifact_root.join(&configmap.path), &configmap.document)?; + for secret in &seed_secrets { + write_yaml_artifact(artifact_root.join(&secret.path), &secret.document)?; } + write_yaml_artifact( + artifact_root.join(KUBERNETES_CONTROLLER_STATE_PVC_PATH), + &json!({ + "apiVersion": "v1", + "kind": "PersistentVolumeClaim", + "metadata": { + "name": KUBERNETES_CONTROLLER_STATE_PVC_NAME, + "labels": labels, + }, + "spec": { + "accessModes": ["ReadWriteOnce"], + "resources": { + "requests": { + "storage": KUBERNETES_CONTROLLER_STATE_STORAGE_REQUEST, + }, + }, + }, + }), + )?; write_yaml_artifact( artifact_root.join(KUBERNETES_CONTROLLER_SERVICE_ACCOUNT_PATH), &json!({ @@ -139,13 +160,14 @@ pub fn inject_kubernetes_site_controller( artifact_root.join(KUBERNETES_CONTROLLER_DEPLOYMENT_PATH), plan, controller_image, - &seed_configmaps, + &seed_secrets, )?; - let mut resource_paths = seed_configmaps + let mut resource_paths = seed_secrets .iter() - .map(|configmap| configmap.path.as_str()) + .map(|secret| secret.path.as_str()) .collect::>(); resource_paths.extend([ + KUBERNETES_CONTROLLER_STATE_PVC_PATH, KUBERNETES_CONTROLLER_SERVICE_ACCOUNT_PATH, KUBERNETES_CONTROLLER_ROLE_PATH, KUBERNETES_CONTROLLER_ROLE_BINDING_PATH, @@ -233,7 +255,7 @@ fn patch_kubernetes_controller_deployment( path: PathBuf, plan: &SiteControllerPlan, controller_image: &str, - seed_configmaps: &[KubernetesControllerSeedConfigMap], + seed_secrets: &[KubernetesControllerSeedSecret], ) -> Result<()> { let raw = fs::read_to_string(&path) .into_diagnostic() @@ -282,18 +304,24 @@ fn patch_kubernetes_controller_deployment( "-lc", format!( "set -eu - mkdir -p {KUBERNETES_CONTROLLER_STATE_ROOT} \ - {KUBERNETES_CONTROLLER_ARTIFACT_ROOT} -cp \ - {KUBERNETES_CONTROLLER_SEED_ROOT}/site-controller-plan.json \ - {KUBERNETES_CONTROLLER_PLAN_PATH} -cp \ - {KUBERNETES_CONTROLLER_SEED_ROOT}/site-controller-state.json \ - {KUBERNETES_CONTROLLER_STATE_PATH} -cp \ - {KUBERNETES_CONTROLLER_SEED_ROOT}/desired-links.json \ - {KUBERNETES_CONTROLLER_DESIRED_LINKS_PATH} +seed_if_missing() {{ + src=\"$1\" + dest=\"$2\" + if [ ! -e \"$dest\" ]; then + cp \"$src\" \"$dest\" + fi +}} +mkdir -p {KUBERNETES_CONTROLLER_STATE_ROOT} \ + {KUBERNETES_CONTROLLER_ARTIFACT_ROOT} +cp {KUBERNETES_CONTROLLER_SEED_ROOT}/site-controller-plan.json \ + {KUBERNETES_CONTROLLER_PLAN_PATH} +seed_if_missing {KUBERNETES_CONTROLLER_SEED_ROOT}/site-controller-state.json \ + {KUBERNETES_CONTROLLER_STATE_PATH} +seed_if_missing {KUBERNETES_CONTROLLER_SEED_ROOT}/desired-links.json \ + {KUBERNETES_CONTROLLER_DESIRED_LINKS_PATH} if [ -d {KUBERNETES_CONTROLLER_SEED_ROOT}/artifact ]; then + rm -rf {KUBERNETES_CONTROLLER_ARTIFACT_ROOT} + mkdir -p {KUBERNETES_CONTROLLER_ARTIFACT_ROOT} cp -R {KUBERNETES_CONTROLLER_SEED_ROOT}/artifact/. \ {KUBERNETES_CONTROLLER_ARTIFACT_ROOT} fi @@ -350,13 +378,13 @@ fi .or_insert_with(|| serde_yaml::Value::Sequence(Vec::new())) .as_sequence_mut() .ok_or_else(|| miette::miette!("controller deployment volumes must be a sequence"))?; - let seed_sources = seed_configmaps + let seed_sources = seed_secrets .iter() - .map(|configmap| { + .map(|secret| { json!({ - "configMap": { - "name": configmap.name.clone(), - "items": configmap.items.clone(), + "secret": { + "name": secret.name.clone(), + "items": secret.items.clone(), } }) }) @@ -378,7 +406,9 @@ fi KUBERNETES_CONTROLLER_STATE_VOLUME, serde_yaml::to_value(json!({ "name": KUBERNETES_CONTROLLER_STATE_VOLUME, - "emptyDir": {} + "persistentVolumeClaim": { + "claimName": KUBERNETES_CONTROLLER_STATE_PVC_NAME, + } })) .into_diagnostic() .wrap_err("failed to serialize controller state volume")?, @@ -409,11 +439,11 @@ fn build_embedded_kubernetes_controller_plan(plan: &SiteControllerPlan) -> SiteC embedded } -fn build_kubernetes_controller_seed_configmaps( +fn build_kubernetes_controller_seed_secrets( artifact_root: &Path, plan: &SiteControllerPlan, embedded_plan: &SiteControllerPlan, -) -> Result> { +) -> Result> { let seed_files = build_kubernetes_controller_seed_files(artifact_root, plan, embedded_plan)?; let mut chunks = Vec::>::new(); let mut current = Vec::<(String, String)>::new(); @@ -421,7 +451,7 @@ fn build_kubernetes_controller_seed_configmaps( for (path, contents) in seed_files { let entry_size = path.len() + contents.len(); if !current.is_empty() - && current_size + entry_size > KUBERNETES_CONTROLLER_SEED_CONFIGMAP_DATA_LIMIT + && current_size + entry_size > KUBERNETES_CONTROLLER_SEED_SECRET_DATA_LIMIT { chunks.push(current); current = Vec::new(); @@ -436,7 +466,7 @@ fn build_kubernetes_controller_seed_configmaps( chunks .into_iter() .enumerate() - .map(|(index, files)| render_kubernetes_controller_seed_configmap(index, files)) + .map(|(index, files)| render_kubernetes_controller_seed_secret(index, files)) .collect() } @@ -478,42 +508,43 @@ fn build_kubernetes_controller_seed_files( Ok(files.into_iter().collect()) } -fn render_kubernetes_controller_seed_configmap( +fn render_kubernetes_controller_seed_secret( index: usize, files: Vec<(String, String)>, -) -> Result { +) -> Result { let name = if index == 0 { format!("{SITE_CONTROLLER_SERVICE_NAME}-seed") } else { format!("{SITE_CONTROLLER_SERVICE_NAME}-seed-{index}") }; let path = if index == 0 { - KUBERNETES_CONTROLLER_SEED_CONFIGMAP_PATH.to_string() + KUBERNETES_CONTROLLER_SEED_SECRET_PATH.to_string() } else { - format!("{KUBERNETES_CONTROLLER_SEED_CONFIGMAP_PATH_PREFIX}-{index}.yaml") + format!("{KUBERNETES_CONTROLLER_SEED_SECRET_PATH_PREFIX}-{index}.yaml") }; - let mut data = serde_json::Map::new(); + let mut string_data = serde_json::Map::new(); let mut items = Vec::with_capacity(files.len()); for (entry_index, (relative_path, contents)) in files.into_iter().enumerate() { let key = format!("seed-{entry_index:04}"); - data.insert(key.clone(), serde_json::Value::String(contents)); + string_data.insert(key.clone(), serde_json::Value::String(contents)); items.push(json!({ "key": key, "path": relative_path, })); } - Ok(KubernetesControllerSeedConfigMap { + Ok(KubernetesControllerSeedSecret { path, name: name.clone(), items, document: json!({ "apiVersion": "v1", - "kind": "ConfigMap", + "kind": "Secret", "metadata": { "name": name, "labels": kubernetes_controller_labels(), }, - "data": data, + "type": "Opaque", + "stringData": string_data, }), }) } @@ -1018,9 +1049,9 @@ spec: .expect("desired links should write"); } - fn seed_configmap_paths(artifact_root: &Path) -> Vec { + fn seed_secret_paths(artifact_root: &Path) -> Vec { let mut paths = - walk_files(&artifact_root.join("01-configmaps")).expect("seed configmaps should list"); + walk_files(&artifact_root.join("01-secrets")).expect("seed secrets should list"); paths.sort(); paths } @@ -1129,15 +1160,15 @@ spec: .and_then(serde_yaml::Value::as_mapping) .and_then(|projected| projected.get(yaml_string("sources"))) .and_then(serde_yaml::Value::as_sequence) - .expect("controller seed volume should project configmaps"); + .expect("controller seed volume should project secrets"); let projected_paths = projected_sources .iter() .flat_map(|source| { source .as_mapping() - .and_then(|mapping| mapping.get(yaml_string("configMap"))) + .and_then(|mapping| mapping.get(yaml_string("secret"))) .and_then(serde_yaml::Value::as_mapping) - .and_then(|config_map| config_map.get(yaml_string("items"))) + .and_then(|secret| secret.get(yaml_string("items"))) .and_then(serde_yaml::Value::as_sequence) .into_iter() .flatten() @@ -1170,29 +1201,53 @@ spec: .iter() .any(|path| path == "artifact/kustomization.yaml") ); + let state_volume = volumes + .iter() + .find(|volume| { + volume + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str) + .is_some_and(|name| name == KUBERNETES_CONTROLLER_STATE_VOLUME) + }) + .and_then(serde_yaml::Value::as_mapping) + .expect("controller deployment should include the state volume"); + assert_eq!( + state_volume + .get(yaml_string("persistentVolumeClaim")) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|pvc| pvc.get(yaml_string("claimName"))) + .and_then(serde_yaml::Value::as_str), + Some(KUBERNETES_CONTROLLER_STATE_PVC_NAME), + "controller state must be backed by a PVC" + ); + assert!( + !state_volume.contains_key(yaml_string("emptyDir")), + "controller state must not be backed by emptyDir" + ); assert!( - seed_configmap_paths(artifact_root) + seed_secret_paths(artifact_root) .iter() .any(|path| path.file_name().and_then(|name| name.to_str()) == Some("amber-site-controller-seed.yaml")), - "kustomization should still include the primary controller seed configmap" + "kustomization should include the primary controller seed secret" ); } #[test] - fn inject_kubernetes_site_controller_chunks_seed_configmaps_to_avoid_annotation_limits() { + fn inject_kubernetes_site_controller_chunks_seed_secrets_to_avoid_object_limits() { let temp = tempfile::tempdir().expect("temp dir"); let artifact_root = temp.path(); write_kubernetes_controller_fixture(artifact_root); fs::create_dir_all(artifact_root.join("seed-data")).expect("seed data dir"); fs::write( artifact_root.join("seed-data/large-a.txt"), - "a".repeat(KUBERNETES_CONTROLLER_SEED_CONFIGMAP_DATA_LIMIT / 2), + "a".repeat(KUBERNETES_CONTROLLER_SEED_SECRET_DATA_LIMIT / 2), ) .expect("large artifact A should write"); fs::write( artifact_root.join("seed-data/large-b.txt"), - "b".repeat(KUBERNETES_CONTROLLER_SEED_CONFIGMAP_DATA_LIMIT / 2), + "b".repeat(KUBERNETES_CONTROLLER_SEED_SECRET_DATA_LIMIT / 2), ) .expect("large artifact B should write"); @@ -1206,20 +1261,20 @@ spec: ) .expect("kubernetes site controller injection should succeed"); - let seed_paths = seed_configmap_paths(artifact_root); + let seed_paths = seed_secret_paths(artifact_root); assert!( seed_paths.len() > 1, - "large controller seeds should be split across multiple configmaps" + "large controller seeds should be split across multiple secrets" ); for seed_path in &seed_paths { - let raw = fs::read_to_string(seed_path).expect("seed configmap should read"); + let raw = fs::read_to_string(seed_path).expect("seed secret should read"); let document: serde_yaml::Value = - serde_yaml::from_str(&raw).expect("seed configmap should parse"); + serde_yaml::from_str(&raw).expect("seed secret should parse"); let data_size = document .as_mapping() - .and_then(|root| root.get(yaml_string("data"))) + .and_then(|root| root.get(yaml_string("stringData"))) .and_then(serde_yaml::Value::as_mapping) - .expect("seed configmap should have data") + .expect("seed secret should have stringData") .iter() .map(|(key, value)| { key.as_str().unwrap_or_default().len() @@ -1227,8 +1282,8 @@ spec: }) .sum::(); assert!( - data_size <= KUBERNETES_CONTROLLER_SEED_CONFIGMAP_DATA_LIMIT, - "seed configmap {} should stay under the chunk limit", + data_size <= KUBERNETES_CONTROLLER_SEED_SECRET_DATA_LIMIT, + "seed secret {} should stay under the chunk limit", seed_path.display() ); } @@ -1256,11 +1311,11 @@ spec: .and_then(serde_yaml::Value::as_mapping) .and_then(|projected| projected.get(yaml_string("sources"))) .and_then(serde_yaml::Value::as_sequence) - .expect("controller deployment should project seed configmaps"); + .expect("controller deployment should project seed secrets"); assert_eq!( projected_sources.len(), seed_paths.len(), - "deployment should project every generated seed configmap" + "deployment should project every generated seed secret" ); } } diff --git a/runtime/site-controller/src/tests.rs b/runtime/site-controller/src/tests.rs index a784726f..1cf1be95 100644 --- a/runtime/site-controller/src/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -10153,13 +10153,26 @@ volumes: command[1].as_str(), Some("/amber/site/state/site-controller-plan.json") ); + #[cfg(unix)] + { + let expected_user = format!("{}:{}", unsafe { libc::getuid() }, unsafe { + libc::getgid() + }); + assert_eq!( + service + .get(serde_yaml::Value::String("user".to_string())) + .and_then(serde_yaml::Value::as_str), + Some(expected_user.as_str()), + "compose site controller should run as the launching user so its host bind-mounted \ + state directory can stay private" + ); + } + #[cfg(not(unix))] assert_eq!( service .get(serde_yaml::Value::String("user".to_string())) .and_then(serde_yaml::Value::as_str), - Some("0:0"), - "compose site controller must stay root so it can access the router-control volume, which \ - compose initializes as a root-owned mount point for the router runtime" + Some("0:0") ); let volumes = service .get(serde_yaml::Value::String("volumes".to_string())) @@ -10208,8 +10221,8 @@ volumes: group_add .iter() .any(|value| value.as_str() == Some("65532")), - "compose site controller runs with dropped capabilities, so it needs the router runtime \ - group to access the router-control socket" + "compose site controller runs as the launching user, so it needs the router runtime group \ + to access the router-control socket" ); assert!( extra_hosts @@ -10400,7 +10413,8 @@ spec: let kustomization = fs::read_to_string(artifact_root.join("kustomization.yaml")) .expect("kustomization should read"); - assert!(kustomization.contains("01-configmaps/amber-site-controller-seed.yaml")); + assert!(kustomization.contains("01-secrets/amber-site-controller-seed.yaml")); + assert!(kustomization.contains("03-persistentvolumeclaims/amber-site-controller-state.yaml")); assert!(kustomization.contains("03-deployments/amber-site-controller.yaml")); assert!(kustomization.contains("04-services/amber-site-controller.yaml")); @@ -10458,9 +10472,10 @@ spec: Some("/amber/site/state/site-controller-plan.json") ); - let seed = - fs::read_to_string(artifact_root.join("01-configmaps/amber-site-controller-seed.yaml")) - .expect("seed configmap should read"); + let seed = fs::read_to_string(artifact_root.join("01-secrets/amber-site-controller-seed.yaml")) + .expect("seed secret should read"); + assert!(seed.contains("kind: Secret")); + assert!(seed.contains("stringData:")); assert!(seed.contains("site-controller-plan.json")); assert!( !seed.contains("artifact.tar.b64"), @@ -10470,6 +10485,14 @@ spec: deployment_raw.contains("cp -R /amber/seed/artifact/. /amber/site/artifact"), "kubernetes controller bootstrap should copy seeded artifact files directly" ); + assert!( + deployment_raw.contains("seed_if_missing"), + "kubernetes controller bootstrap should not overwrite mutable state on pod recreation" + ); + assert!( + deployment_raw.contains("persistentVolumeClaim"), + "kubernetes controller state should be backed by a PVC" + ); assert!(seed.contains(&format!("http://amber-site-controller:{controller_port}"))); assert!(deployment_raw.contains("EXISTING_ENV")); From cb6bbca459361277e66b208e03663d638e269ebd Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Mon, 11 May 2026 14:42:41 +0800 Subject: [PATCH 24/25] Use route identity for dynamic caps MCP auth --- cli/src/command_support.rs | 14 - cli/src/mixed_run/launch_bundle.rs | 5 - .../site-controller/src/control_state_mcp.rs | 183 ++++++---- .../src/default_runtime/child_runtime.rs | 2 - .../src/default_runtime/compose_controller.rs | 1 - .../default_runtime/kubernetes_controller.rs | 1 - runtime/site-controller/src/http.rs | 17 +- runtime/site-controller/src/lib.rs | 6 +- runtime/site-controller/src/planner.rs | 1 - .../site-controller/src/site_controller.rs | 27 +- runtime/site-controller/src/state.rs | 14 - runtime/site-controller/src/tests.rs | 313 +++++++++++------- 12 files changed, 336 insertions(+), 248 deletions(-) diff --git a/cli/src/command_support.rs b/cli/src/command_support.rs index 45630405..fd543eda 100644 --- a/cli/src/command_support.rs +++ b/cli/src/command_support.rs @@ -713,8 +713,6 @@ fn materialize_standalone_site_controller_export( ); let local_router_control = crate::mixed_run::site_controller_local_router_control(export.kind, &artifact_root); - let control_state_auth_token = - standalone_control_state_auth_token(&run_plan.mesh_scope, run_id, "site-controller"); let router_mesh_port = Some(24000); if let Some(port) = router_mesh_port && export.kind == SiteKind::Compose @@ -747,7 +745,6 @@ fn materialize_standalone_site_controller_export( &plan_state_root, &plan_site_state_root, &plan_artifact_root, - &control_state_auth_token, controller_identity_path.as_deref(), None, None, @@ -806,17 +803,6 @@ fn compose_mount_source_relative_to_artifact(artifact_root: &Path, path: &Path) Ok(format!("./{relative}")) } -fn standalone_control_state_auth_token(mesh_scope: &str, run_id: &str, purpose: &str) -> String { - base64::engine::general_purpose::STANDARD.encode( - amber_mesh::MeshIdentity::derive( - format!("/framework/{purpose}"), - Some(mesh_scope.to_string()), - &format!("standalone-control-state:{run_id}"), - ) - .public_key, - ) -} - fn standalone_controller_temp_state_root(site_id: &str) -> Result { let nanos = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) diff --git a/cli/src/mixed_run/launch_bundle.rs b/cli/src/mixed_run/launch_bundle.rs index b1932831..9def352f 100644 --- a/cli/src/mixed_run/launch_bundle.rs +++ b/cli/src/mixed_run/launch_bundle.rs @@ -213,10 +213,6 @@ pub(super) fn materialize_launch_bundle( write_json(&run_plan_path, run_plan)?; let site_controller_image = site_controller_image_reference()?; - let control_state_auth_token = amber_site_controller::generate_control_state_auth_token( - &run_plan.mesh_scope, - "site-controller", - ); let observability = materialize_observability(bundle_root, run_id, &run_plan.mesh_scope, observability)?; let observability_endpoint = observability @@ -460,7 +456,6 @@ pub(super) fn materialize_launch_bundle( &state_root, &site.site_state_root, &site.artifact_dir, - &control_state_auth_token, site_controller_identity_path( site.site_plan.site.kind, &site.artifact_dir, diff --git a/runtime/site-controller/src/control_state_mcp.rs b/runtime/site-controller/src/control_state_mcp.rs index 54ed6d84..e23d69e2 100644 --- a/runtime/site-controller/src/control_state_mcp.rs +++ b/runtime/site-controller/src/control_state_mcp.rs @@ -27,7 +27,9 @@ use super::{ mcp_common::{McpOperationResponse, json_response, map_protocol_api_error}, planner::SiteControllerApp, site_controller::{ - execute_site_controller_dynamic_caps_inspect, execute_site_controller_dynamic_caps_mutate, + DynamicCapsRequestAuth, authorize_dynamic_caps_mcp_session_request, + authorize_dynamic_caps_request, execute_site_controller_dynamic_caps_inspect, + execute_site_controller_dynamic_caps_mutate, }, }; @@ -58,16 +60,35 @@ impl FrameworkDynamicCapsMcp { } } - fn authorize(&self, context: &RequestContext) -> Result<(), McpError> { - let parts = context + fn http_parts<'a>( + &self, + context: &'a RequestContext, + ) -> Result<&'a Parts, McpError> { + context .extensions .get::() - .ok_or_else(|| McpError::invalid_request("missing HTTP request context", None))?; - super::http::authorize_control_state_auth_header( - &parts.headers, - self.app.control.control_state_auth_token.as_ref(), - ) - .map_err(map_protocol_api_error) + .ok_or_else(|| McpError::invalid_request("missing HTTP request context", None)) + } + + async fn authorize_session( + &self, + context: &RequestContext, + ) -> Result<(), McpError> { + let parts = self.http_parts(context)?; + authorize_dynamic_caps_mcp_session_request(&self.app, &parts.headers) + .await + .map_err(map_protocol_api_error) + } + + async fn authorize_operation( + &self, + context: &RequestContext, + expected_component_id: &str, + ) -> Result { + let parts = self.http_parts(context)?; + authorize_dynamic_caps_request(&self.app, &parts.headers, expected_component_id) + .await + .map_err(map_protocol_api_error) } fn help_resource(&self) -> String { @@ -251,46 +272,56 @@ impl FrameworkDynamicCapsMcp { Parameters(args): Parameters, context: RequestContext, ) -> Result, McpError> { - self.authorize(&context)?; - let response = execute_site_controller_dynamic_caps_inspect( - &self.app, - match args { - InspectArgs::HeldList { - holder_component_id, - } => DynamicCapsInspectRequest::HeldList( - dynamic_caps::ControlDynamicHeldListRequest { - holder_component_id, - }, - ), - InspectArgs::HeldDetail { + let request = match args { + InspectArgs::HeldList { + holder_component_id, + } => DynamicCapsInspectRequest::HeldList(dynamic_caps::ControlDynamicHeldListRequest { + holder_component_id, + }), + InspectArgs::HeldDetail { + holder_component_id, + held_id, + } => DynamicCapsInspectRequest::HeldDetail( + dynamic_caps::ControlDynamicHeldDetailRequest { holder_component_id, held_id, - } => DynamicCapsInspectRequest::HeldDetail( - dynamic_caps::ControlDynamicHeldDetailRequest { - holder_component_id, - held_id, - }, - ), - InspectArgs::InspectRef { + }, + ), + InspectArgs::InspectRef { + holder_component_id, + r#ref, + } => DynamicCapsInspectRequest::InspectRef( + dynamic_caps::ControlDynamicInspectRefRequest { holder_component_id, r#ref, - } => DynamicCapsInspectRequest::InspectRef( - dynamic_caps::ControlDynamicInspectRefRequest { - holder_component_id, - r#ref, - }, - ), - InspectArgs::ResolveOrigin { + }, + ), + InspectArgs::ResolveOrigin { + holder_component_id, + source, + } => DynamicCapsInspectRequest::ResolveOrigin( + dynamic_caps::ControlDynamicResolveOriginRequest { holder_component_id, - source, - } => DynamicCapsInspectRequest::ResolveOrigin( - dynamic_caps::ControlDynamicResolveOriginRequest { - holder_component_id, - source: source.into(), - }, - ), - }, - false, + source: source.into(), + }, + ), + }; + let expected_component_id = match &request { + DynamicCapsInspectRequest::HeldList(request) => request.holder_component_id.as_str(), + DynamicCapsInspectRequest::HeldDetail(request) => request.holder_component_id.as_str(), + DynamicCapsInspectRequest::InspectRef(request) => request.holder_component_id.as_str(), + DynamicCapsInspectRequest::ResolveOrigin(request) => { + request.holder_component_id.as_str() + } + } + .to_string(); + let auth = self + .authorize_operation(&context, &expected_component_id) + .await?; + let response = execute_site_controller_dynamic_caps_inspect( + &self.app, + request, + matches!(auth, DynamicCapsRequestAuth::RemoteController), ) .await .map_err(map_protocol_api_error)?; @@ -313,32 +344,40 @@ impl FrameworkDynamicCapsMcp { Parameters(args): Parameters, context: RequestContext, ) -> Result, McpError> { - self.authorize(&context)?; + let request = match args { + MutateArgs::Share { + caller_component_id, + source, + recipient_component_id, + idempotency_key, + options, + } => DynamicCapsMutateRequest::Share(dynamic_caps::ControlDynamicShareRequest { + caller_component_id, + source: source.into(), + recipient_component_id, + idempotency_key, + options, + }), + MutateArgs::Revoke { + caller_component_id, + target, + } => DynamicCapsMutateRequest::Revoke(dynamic_caps::ControlDynamicRevokeRequest { + caller_component_id, + target: target.into(), + }), + }; + let expected_component_id = match &request { + DynamicCapsMutateRequest::Share(request) => request.caller_component_id.as_str(), + DynamicCapsMutateRequest::Revoke(request) => request.caller_component_id.as_str(), + } + .to_string(); + let auth = self + .authorize_operation(&context, &expected_component_id) + .await?; let response = execute_site_controller_dynamic_caps_mutate( &self.app, - match args { - MutateArgs::Share { - caller_component_id, - source, - recipient_component_id, - idempotency_key, - options, - } => DynamicCapsMutateRequest::Share(dynamic_caps::ControlDynamicShareRequest { - caller_component_id, - source: source.into(), - recipient_component_id, - idempotency_key, - options, - }), - MutateArgs::Revoke { - caller_component_id, - target, - } => DynamicCapsMutateRequest::Revoke(dynamic_caps::ControlDynamicRevokeRequest { - caller_component_id, - target: target.into(), - }), - }, - false, + request, + matches!(auth, DynamicCapsRequestAuth::RemoteController), ) .await .map_err(map_protocol_api_error)?; @@ -372,7 +411,7 @@ impl ServerHandler for FrameworkDynamicCapsMcp { request: InitializeRequestParams, context: RequestContext, ) -> Result { - self.authorize(&context)?; + self.authorize_session(&context).await?; if context.peer.peer_info().is_none() { context.peer.set_peer_info(request); } @@ -384,7 +423,7 @@ impl ServerHandler for FrameworkDynamicCapsMcp { _request: Option, context: RequestContext, ) -> Result { - self.authorize(&context)?; + self.authorize_session(&context).await?; Ok(ListResourcesResult { resources: vec![ RawResource::new(HELP_RESOURCE_URI, "framework dynamic caps MCP").no_annotation(), @@ -399,7 +438,7 @@ impl ServerHandler for FrameworkDynamicCapsMcp { _request: Option, context: RequestContext, ) -> Result { - self.authorize(&context)?; + self.authorize_session(&context).await?; Ok(ListResourceTemplatesResult { resource_templates: vec![ RawResourceTemplate::new( @@ -418,7 +457,7 @@ impl ServerHandler for FrameworkDynamicCapsMcp { request: ReadResourceRequestParams, context: RequestContext, ) -> Result { - self.authorize(&context)?; + self.authorize_session(&context).await?; let uri = request.uri; let text = if uri.as_str() == HELP_RESOURCE_URI { self.help_resource() diff --git a/runtime/site-controller/src/default_runtime/child_runtime.rs b/runtime/site-controller/src/default_runtime/child_runtime.rs index d73be888..9b4931d8 100644 --- a/runtime/site-controller/src/default_runtime/child_runtime.rs +++ b/runtime/site-controller/src/default_runtime/child_runtime.rs @@ -1442,7 +1442,6 @@ mod tests { state_root: state_root.display().to_string(), site_state_root: compose_state_root.display().to_string(), artifact_dir: compose_artifact.display().to_string(), - control_state_auth_token: "test-auth".to_string(), controller_identity_path: None, storage_root: None, runtime_root: None, @@ -1572,7 +1571,6 @@ mod tests { state_root: state_root.display().to_string(), site_state_root: direct_state_root.display().to_string(), artifact_dir: direct_artifact.display().to_string(), - control_state_auth_token: "test-auth".to_string(), controller_identity_path: None, storage_root: Some( temp.path() diff --git a/runtime/site-controller/src/default_runtime/compose_controller.rs b/runtime/site-controller/src/default_runtime/compose_controller.rs index 57aa16a1..647363ca 100644 --- a/runtime/site-controller/src/default_runtime/compose_controller.rs +++ b/runtime/site-controller/src/default_runtime/compose_controller.rs @@ -343,7 +343,6 @@ mod tests { state_root: run_root.join("state-root").display().to_string(), site_state_root: run_root.join("site-state").display().to_string(), artifact_dir: run_root.join("artifact").display().to_string(), - control_state_auth_token: "token".to_string(), controller_identity_path: None, storage_root: None, runtime_root: None, diff --git a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs index 8fde5782..88a6b296 100644 --- a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs +++ b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs @@ -935,7 +935,6 @@ mod tests { state_root: root.join("state-root").display().to_string(), site_state_root: root.join("site-state").display().to_string(), artifact_dir: root.join("artifact").display().to_string(), - control_state_auth_token: "token".to_string(), controller_identity_path: None, storage_root: None, runtime_root: None, diff --git a/runtime/site-controller/src/http.rs b/runtime/site-controller/src/http.rs index e19a7093..44be2691 100644 --- a/runtime/site-controller/src/http.rs +++ b/runtime/site-controller/src/http.rs @@ -8,27 +8,12 @@ use std::{ use miette::{IntoDiagnostic as _, Result, WrapErr as _}; use serde::{Deserialize, Serialize}; -use super::{ - orchestration::ProtocolApiError, planner::ControlStateApp, state::CONTROL_STATE_AUTH_HEADER, *, -}; +use super::{orchestration::ProtocolApiError, planner::ControlStateApp, *}; pub(super) async fn cleanup_dynamic_bridge_proxies(app: &ControlStateApp) -> Result<()> { app.runtime.cleanup().await } -pub(super) fn authorize_control_state_auth_header( - headers: &HeaderMap, - expected: &str, -) -> std::result::Result<(), ProtocolApiError> { - let actual = required_header(headers, CONTROL_STATE_AUTH_HEADER)?; - if actual != expected { - return Err(ProtocolApiError::unauthorized( - "invalid authenticated control-state request header", - )); - } - Ok(()) -} - pub(super) fn required_header( headers: &HeaderMap, name: &str, diff --git a/runtime/site-controller/src/lib.rs b/runtime/site-controller/src/lib.rs index dfd6e4f3..9ea7175a 100644 --- a/runtime/site-controller/src/lib.rs +++ b/runtime/site-controller/src/lib.rs @@ -22,7 +22,7 @@ use amber_manifest::{ Manifest, ManifestRef, ManifestSpans, NetworkProtocol, RawBinding, RawExportTarget, SlotDecl, }; use amber_mesh::{ - MeshIdentity, MeshProtocol, + MeshProtocol, component_protocol::{ BindingInputDescription, ChildDescribeResponse, ChildHandle, ChildListResponse, ChildState, ChildSummary, ConfigFieldDescription, CreateChildRequest, CreateChildResponse, InputState, @@ -100,8 +100,8 @@ pub use self::{ DynamicInputDirectRecord, DynamicInputRouteTarget, DynamicProxyExportRecord, SITE_CONTROLLER_INTERNAL_CAPABILITY, SITE_CONTROLLER_MESH_IDENTITY_PATH, SITE_CONTROLLER_SERVICE_NAME, SiteControllerPlan, authority_url_for_listen_addr, - build_site_controller_state, generate_control_state_auth_token, - site_controller_internal_route_id, write_control_state, write_site_controller_plan, + build_site_controller_state, site_controller_internal_route_id, write_control_state, + write_site_controller_plan, }, }; diff --git a/runtime/site-controller/src/planner.rs b/runtime/site-controller/src/planner.rs index e53fc623..e55177da 100644 --- a/runtime/site-controller/src/planner.rs +++ b/runtime/site-controller/src/planner.rs @@ -3550,7 +3550,6 @@ pub(super) struct ControlStateApp { pub(super) run_root: PathBuf, pub(super) state_root: PathBuf, pub(super) mesh_scope: Arc, - pub(super) control_state_auth_token: Arc, pub(super) controller_plan: Arc, pub(super) authority_locks: Arc>>>>, pub(super) runtime: SharedSiteControllerRuntime, diff --git a/runtime/site-controller/src/site_controller.rs b/runtime/site-controller/src/site_controller.rs index 374a8348..615b89d0 100644 --- a/runtime/site-controller/src/site_controller.rs +++ b/runtime/site-controller/src/site_controller.rs @@ -75,7 +75,6 @@ pub(crate) async fn run_site_controller( run_root: PathBuf::from(&plan.run_root), state_root: PathBuf::from(&plan.state_root), mesh_scope: Arc::::from(plan.mesh_scope.clone()), - control_state_auth_token: Arc::::from(plan.control_state_auth_token.clone()), controller_plan: Arc::new(plan.clone()), authority_locks: Arc::new(Mutex::new(BTreeMap::new())), runtime, @@ -491,6 +490,32 @@ pub(super) async fn authorize_dynamic_caps_request( Ok(DynamicCapsRequestAuth::RemoteController) } +pub(super) async fn authorize_dynamic_caps_mcp_session_request( + app: &SiteControllerApp, + headers: &HeaderMap, +) -> std::result::Result<(), ProtocolApiError> { + let _route_id = required_header(headers, FRAMEWORK_ROUTE_ID_HEADER)?; + let peer_id = required_header(headers, FRAMEWORK_PEER_ID_HEADER)?; + if authorize_remote_controller_request(app, headers) + .await + .is_ok() + { + return Ok(()); + } + + let component_id = dynamic_caps::logical_component_id(&peer_id); + let state = app.control.control_state.lock().await.clone(); + if !dynamic_caps::live_component_ids(&state) + .map_err(ProtocolApiError::from)? + .contains(&component_id) + { + return Err(ProtocolApiError::unauthorized(format!( + "dynamic capability MCP request came from non-component peer `{peer_id}`" + ))); + } + authorize_dynamic_caps_sidecar_request(headers, &component_id) +} + fn ensure_controller_ready(app: &SiteControllerApp) -> std::result::Result<(), ProtocolApiError> { if app.ready.load(std::sync::atomic::Ordering::SeqCst) { return Ok(()); diff --git a/runtime/site-controller/src/state.rs b/runtime/site-controller/src/state.rs index aee1437a..1bd280cc 100644 --- a/runtime/site-controller/src/state.rs +++ b/runtime/site-controller/src/state.rs @@ -8,7 +8,6 @@ pub(super) const SITE_CONTROLLER_PLAN_VERSION: u32 = 1; pub(super) const SITE_CONTROLLER_STATE_PATH: &str = "/v1/controller/state"; pub(super) const FRAMEWORK_ROUTE_ID_HEADER: &str = "x-amber-route-id"; pub(super) const FRAMEWORK_PEER_ID_HEADER: &str = "x-amber-peer-id"; -pub(super) const CONTROL_STATE_AUTH_HEADER: &str = "x-amber-control-state-auth"; pub const SITE_CONTROLLER_INTERNAL_CAPABILITY: &str = amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME; pub const SITE_CONTROLLER_SERVICE_NAME: &str = "amber-site-controller"; @@ -274,7 +273,6 @@ pub struct SiteControllerPlan { pub state_root: String, pub site_state_root: String, pub artifact_dir: String, - pub control_state_auth_token: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub controller_identity_path: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1062,16 +1060,6 @@ fn remap_output_handle_for_snapshot( }) } -pub fn generate_control_state_auth_token(mesh_scope: &str, purpose: &str) -> String { - base64::engine::general_purpose::STANDARD.encode( - MeshIdentity::generate( - format!("/framework/{purpose}"), - Some(mesh_scope.to_string()), - ) - .public_key, - ) -} - pub fn authority_url_for_listen_addr(listen_addr: SocketAddr) -> String { let dial_addr = if listen_addr.ip().is_unspecified() { SocketAddr::from(([127, 0, 0, 1], listen_addr.port())) @@ -1238,7 +1226,6 @@ pub fn write_site_controller_plan( state_root: &Path, site_state_root: &Path, artifact_dir: &Path, - control_state_auth_token: &str, controller_identity_path: Option<&str>, storage_root: Option<&str>, runtime_root: Option<&str>, @@ -1272,7 +1259,6 @@ pub fn write_site_controller_plan( state_root: state_root.display().to_string(), site_state_root: site_state_root.display().to_string(), artifact_dir: artifact_dir.display().to_string(), - control_state_auth_token: control_state_auth_token.to_string(), controller_identity_path: controller_identity_path.map(str::to_string), storage_root: storage_root.map(str::to_string), runtime_root: runtime_root.map(str::to_string), diff --git a/runtime/site-controller/src/tests.rs b/runtime/site-controller/src/tests.rs index 1cf1be95..2a853d6c 100644 --- a/runtime/site-controller/src/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -836,7 +836,6 @@ fn with_runtime(app: &ControlStateApp, runtime: SharedSiteControllerRuntime) -> run_root: app.run_root.clone(), state_root: app.state_root.clone(), mesh_scope: app.mesh_scope.clone(), - control_state_auth_token: app.control_state_auth_token.clone(), controller_plan: app.controller_plan.clone(), authority_locks: app.authority_locks.clone(), runtime, @@ -2723,7 +2722,6 @@ fn test_control_state_app( run_root: run_root.clone(), state_root: state_root.clone(), mesh_scope: Arc::::from("test-mesh"), - control_state_auth_token: Arc::::from("test-control-state-auth"), controller_plan: Arc::new(SiteControllerPlan { schema: "amber.framework_component.site_controller_plan".to_string(), version: 1, @@ -2746,7 +2744,6 @@ fn test_control_state_app( state_root: state_root.display().to_string(), site_state_root: site_state_root.display().to_string(), artifact_dir: artifact_dir.display().to_string(), - control_state_auth_token: "test-control-state-auth".to_string(), controller_identity_path: None, storage_root: Some(storage_root.display().to_string()), runtime_root: Some(runtime_root.display().to_string()), @@ -2782,6 +2779,10 @@ fn sse_json_rpc_message(body: &str) -> Value { .unwrap_or_else(|err| panic!("parse JSON-RPC payload from SSE: {err}; {payload}")) } +fn json_rpc_message(body: &str) -> Value { + serde_json::from_str(body).unwrap_or_else(|_| sse_json_rpc_message(body)) +} + async fn spawn_test_router(router: Router) -> (String, tokio::task::JoinHandle<()>) { let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) .await @@ -2939,7 +2940,6 @@ fn with_controller_endpoint( run_root: app.run_root.clone(), state_root: app.state_root.clone(), mesh_scope: app.mesh_scope.clone(), - control_state_auth_token: app.control_state_auth_token.clone(), controller_plan: Arc::new(controller_plan), authority_locks: app.authority_locks.clone(), runtime: app.runtime.clone(), @@ -3202,37 +3202,8 @@ impl TestMcpClient { headers: Vec<(String, String)>, ) -> Self { let client = Client::new(); - let initialize = json!({ - "jsonrpc": "2.0", - "id": 0, - "method": "initialize", - "params": { - "protocolVersion": "2025-06-18", - "capabilities": {}, - "clientInfo": { - "name": client_name, - "version": "0.0.0", - }, - }, - }); - let response = with_test_timeout( - format!("MCP initialize request to {endpoint}"), - apply_headers(client.post(endpoint), &headers) - .header("content-type", "application/json") - .header("accept", "application/json, text/event-stream") - .json(&initialize) - .send(), - ) - .await - .expect("send initialize request"); - let status = response.status(); - let response_headers = response.headers().clone(); - let body = with_test_timeout( - format!("read MCP initialize response from {endpoint}"), - response.text(), - ) - .await - .expect("read initialize response"); + let (status, response_headers, payload, body) = + Self::initialize_response(&client, endpoint, client_name, &headers).await; assert_eq!(status, StatusCode::OK, "initialize failed: {body}"); let session_id = response_headers .get("mcp-session-id") @@ -3240,7 +3211,6 @@ impl TestMcpClient { .to_str() .expect("session ID should be valid UTF-8") .to_string(); - let payload = sse_json_rpc_message(&body); assert!( payload.get("error").is_none(), "initialize returned error: {payload:#?}" @@ -3275,6 +3245,47 @@ impl TestMcpClient { } } + async fn initialize_response( + client: &Client, + endpoint: &str, + client_name: &str, + headers: &[(String, String)], + ) -> (StatusCode, HeaderMap, Value, String) { + let initialize = json!({ + "jsonrpc": "2.0", + "id": 0, + "method": "initialize", + "params": { + "protocolVersion": "2025-06-18", + "capabilities": {}, + "clientInfo": { + "name": client_name, + "version": "0.0.0", + }, + }, + }); + let response = with_test_timeout( + format!("MCP initialize request to {endpoint}"), + apply_headers(client.post(endpoint), headers) + .header("content-type", "application/json") + .header("accept", "application/json, text/event-stream") + .json(&initialize) + .send(), + ) + .await + .expect("send initialize request"); + let status = response.status(); + let response_headers = response.headers().clone(); + let body = with_test_timeout( + format!("read MCP initialize response from {endpoint}"), + response.text(), + ) + .await + .expect("read initialize response"); + let payload = json_rpc_message(&body); + (status, response_headers, payload, body) + } + async fn tools_list(&mut self) -> Vec { self.request("tools/list", json!({})) .await @@ -3329,7 +3340,35 @@ impl TestMcpClient { .unwrap_or_else(|err| panic!("deserialize tool result for {name}: {err}; {result:#?}")) } + async fn call_tool_error(&mut self, name: &str, arguments: Value) -> Value { + let payload = self + .request_payload( + "tools/call", + json!({ + "name": name, + "arguments": arguments, + }), + ) + .await; + payload + .get("error") + .cloned() + .unwrap_or_else(|| panic!("tool {name} unexpectedly succeeded: {payload:#?}")) + } + async fn request(&mut self, method: &str, params: Value) -> Value { + let payload = self.request_payload(method, params).await; + assert!( + payload.get("error").is_none(), + "MCP request {method} returned error: {payload:#?}" + ); + payload + .get("result") + .cloned() + .expect("MCP response should include result") + } + + async fn request_payload(&mut self, method: &str, params: Value) -> Value { let id = self.next_id; self.next_id += 1; let response = apply_headers( @@ -3367,14 +3406,7 @@ impl TestMcpClient { ); let payload = sse_json_rpc_message(&body); assert_eq!(payload["id"].as_u64(), Some(id)); - assert!( - payload.get("error").is_none(), - "MCP request {method} returned error: {payload:#?}" - ); payload - .get("result") - .cloned() - .expect("MCP response should include result") } } @@ -3689,7 +3721,6 @@ async fn framework_component_rejects_stale_nonlocal_controller_delivery() { run_root: app.run_root.clone(), state_root: app.state_root.clone(), mesh_scope: app.mesh_scope.clone(), - control_state_auth_token: app.control_state_auth_token.clone(), controller_plan: Arc::new(controller_plan), authority_locks: app.authority_locks.clone(), runtime: app.runtime.clone(), @@ -3973,7 +4004,7 @@ struct DynamicCapsMcpHarness { _dir: TempDir, client: Client, base_url: String, - control_state_auth_token: String, + controller_route_id: String, handles: Vec>, } @@ -3993,6 +4024,25 @@ impl DynamicCapsMcpHarness { &base_url, listen_addr, ); + let controller_route_id = { + let state = app.control_state.lock().await; + let scenario = Scenario::try_from(state.base_scenario.clone()).expect("base scenario"); + let controller = scenario + .components_iter() + .find(|(_, component)| { + amber_compiler::run_plan::framework_component_controller_metadata( + component.metadata.as_ref(), + ) + .is_some_and(|metadata| metadata.execution_site == app.controller_plan.site_id) + }) + .map(|(_, component)| component) + .expect("site controller component should be injected"); + amber_mesh::component_route_id( + controller.moniker.as_str(), + amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, + amber_mesh::MeshProtocol::Http, + ) + }; let mut handles = vec![install_dynamic_caps_origin_fixture(&app).await]; let controller_app = SiteControllerApp { control: app, @@ -4010,16 +4060,22 @@ impl DynamicCapsMcpHarness { _dir: dir, client: Client::new(), base_url, - control_state_auth_token: "test-control-state-auth".to_string(), + controller_route_id, handles, } } - fn mcp_headers(&self) -> Vec<(String, String)> { - vec![( - CONTROL_STATE_AUTH_HEADER.to_string(), - self.control_state_auth_token.clone(), - )] + fn dynamic_caps_headers_for_component(&self, component_id: &str) -> Vec<(String, String)> { + let peer_id = dynamic_caps::moniker_from_logical_component_id(component_id) + .expect("logical component id should map to a moniker") + .to_string(); + vec![ + ( + FRAMEWORK_ROUTE_ID_HEADER.to_string(), + self.controller_route_id.clone(), + ), + (FRAMEWORK_PEER_ID_HEADER.to_string(), peer_id), + ] } fn dynamic_caps_http_headers(&self, body: &Req) -> Vec<(String, String)> { @@ -4029,29 +4085,38 @@ impl DynamicCapsMcpHarness { .or_else(|| body.get("caller_component_id")) .and_then(Value::as_str) .expect("dynamic caps control requests should identify the caller component"); - let peer_id = dynamic_caps::moniker_from_logical_component_id(component_id) - .expect("logical component id should map to a moniker") - .to_string(); - let route_id = amber_mesh::component_route_id( - peer_id.as_str(), - amber_mesh::FRAMEWORK_COMPONENT_CONTROLLER_INTERNAL_PROVIDE_NAME, - amber_mesh::MeshProtocol::Http, - ); - vec![ - (FRAMEWORK_ROUTE_ID_HEADER.to_string(), route_id), - (FRAMEWORK_PEER_ID_HEADER.to_string(), peer_id), - ] + self.dynamic_caps_headers_for_component(component_id) } - async fn connect(&self) -> TestMcpClient { + async fn connect_as(&self, component_id: &str) -> TestMcpClient { TestMcpClient::connect_endpoint( &format!("{}/v1/controller/dynamic-caps/mcp", self.base_url), "framework-dynamic-caps-test", - self.mcp_headers(), + self.dynamic_caps_headers_for_component(component_id), ) .await } + async fn connect(&self) -> TestMcpClient { + self.connect_as("components./alice").await + } + + async fn initialize_with_headers(&self, headers: Vec<(String, String)>) -> (StatusCode, Value) { + let client = Client::new(); + let (status, _response_headers, payload, body) = TestMcpClient::initialize_response( + &client, + &format!("{}/v1/controller/dynamic-caps/mcp", self.base_url), + "framework-dynamic-caps-test", + &headers, + ) + .await; + assert!( + status.is_success(), + "initialize returned transport failure {status}: {body}" + ); + (status, payload) + } + async fn post_json(&self, path: &str, body: &Req) -> T { http_post_json( &self.client, @@ -4890,11 +4955,63 @@ async fn dynamic_caps_mcp_discovers_compact_surface() { ); } +#[tokio::test] +async fn dynamic_caps_mcp_rejects_control_state_auth_without_route_identity() { + let harness = DynamicCapsMcpHarness::start().await; + let (_status, payload) = harness + .initialize_with_headers(vec![( + "x-amber-control-state-auth".to_string(), + "test-control-state-auth".to_string(), + )]) + .await; + + let error = payload + .get("error") + .unwrap_or_else(|| panic!("initialize unexpectedly succeeded: {payload:#?}")); + assert!( + error.to_string().contains(FRAMEWORK_ROUTE_ID_HEADER), + "dynamic-caps MCP must require router-injected route identity, got {error:#?}" + ); +} + +#[tokio::test] +async fn dynamic_caps_mcp_rejects_tool_calls_for_a_different_component_identity() { + let harness = DynamicCapsMcpHarness::start().await; + let mut alice_mcp = harness.connect_as("components./alice").await; + + let error = alice_mcp + .call_tool_error( + "amber.v1.framework_dynamic_caps.inspect", + json!({ + "op": "held_list", + "holder_component_id": "components./carol", + }), + ) + .await; + + assert_eq!( + error + .get("data") + .and_then(|data| data.get("code")) + .and_then(Value::as_str), + Some("unauthorized"), + "dynamic-caps MCP must reject cross-component tool arguments, got {error:#?}" + ); + assert!( + error + .to_string() + .contains("unknown framework capability instance") + || error.to_string().contains("components./carol"), + "dynamic-caps MCP should fail before executing a cross-component operation, got {error:#?}" + ); +} + #[tokio::test] async fn dynamic_caps_mcp_matches_http_surface() { let http = DynamicCapsMcpHarness::start().await; let mcp_harness = DynamicCapsMcpHarness::start().await; - let mut mcp = mcp_harness.connect().await; + let mut alice_mcp = mcp_harness.connect_as("components./alice").await; + let mut carol_mcp = mcp_harness.connect_as("components./carol").await; let held_list_request = dynamic_caps::ControlDynamicHeldListRequest { holder_component_id: "components./alice".to_string(), @@ -4902,7 +5019,7 @@ async fn dynamic_caps_mcp_matches_http_surface() { let http_held: amber_mesh::dynamic_caps::HeldListResponse = http .post_json("/v1/controller/dynamic-caps/held", &held_list_request) .await; - let mcp_held: Value = mcp + let mcp_held: Value = alice_mcp .call_tool( "amber.v1.framework_dynamic_caps.inspect", json!({ @@ -4932,7 +5049,7 @@ async fn dynamic_caps_mcp_matches_http_surface() { &held_detail_request, ) .await; - let mcp_detail: Value = mcp + let mcp_detail: Value = alice_mcp .call_tool( "amber.v1.framework_dynamic_caps.inspect", json!({ @@ -4964,7 +5081,7 @@ async fn dynamic_caps_mcp_matches_http_surface() { let http_share: amber_mesh::dynamic_caps::ShareResponse = http .post_json("/v1/controller/dynamic-caps/share", &share_request) .await; - let mcp_share: Value = mcp + let mcp_share: Value = alice_mcp .call_tool( "amber.v1.framework_dynamic_caps.mutate", json!({ @@ -5005,7 +5122,7 @@ async fn dynamic_caps_mcp_matches_http_surface() { let http_carol_held: amber_mesh::dynamic_caps::HeldListResponse = http .post_json("/v1/controller/dynamic-caps/held", &carol_held_request) .await; - let mcp_carol_held: Value = mcp + let mcp_carol_held: Value = carol_mcp .call_tool( "amber.v1.framework_dynamic_caps.inspect", json!({ @@ -5029,7 +5146,7 @@ async fn dynamic_caps_mcp_matches_http_surface() { &inspect_ref_request, ) .await; - let mcp_inspect_ref: Value = mcp + let mcp_inspect_ref: Value = carol_mcp .call_tool( "amber.v1.framework_dynamic_caps.inspect", json!({ @@ -5056,7 +5173,7 @@ async fn dynamic_caps_mcp_matches_http_surface() { &resolve_origin_request, ) .await; - let mcp_resolve_origin: Value = mcp + let mcp_resolve_origin: Value = alice_mcp .call_tool( "amber.v1.framework_dynamic_caps.inspect", json!({ @@ -5084,7 +5201,7 @@ async fn dynamic_caps_mcp_matches_http_surface() { let http_revoke: amber_mesh::dynamic_caps::RevokeResponse = http .post_json("/v1/controller/dynamic-caps/revoke", &revoke_request) .await; - let mcp_revoke: Value = mcp + let mcp_revoke: Value = alice_mcp .call_tool( "amber.v1.framework_dynamic_caps.mutate", json!({ @@ -5112,7 +5229,7 @@ async fn dynamic_caps_mcp_matches_http_surface() { &revoked_detail_request, ) .await; - let mcp_revoked_detail: Value = mcp + let mcp_revoked_detail: Value = carol_mcp .call_tool( "amber.v1.framework_dynamic_caps.inspect", json!({ @@ -6959,29 +7076,6 @@ async fn destroy_and_recreate_same_child_name_revokes_then_restores_stable_frame .expect("recreated child should regain framework authorization"); } -#[test] -fn control_state_auth_header_must_match_expected_token() { - let mut headers = HeaderMap::new(); - let missing = authorize_control_state_auth_header(&headers, "expected") - .expect_err("missing auth header should be rejected"); - assert_eq!(missing.0.code, ProtocolErrorCode::Unauthorized); - - headers.insert( - CONTROL_STATE_AUTH_HEADER, - "wrong".parse().expect("header should parse"), - ); - let wrong = authorize_control_state_auth_header(&headers, "expected") - .expect_err("mismatched auth header should be rejected"); - assert_eq!(wrong.0.code, ProtocolErrorCode::Unauthorized); - - headers.insert( - CONTROL_STATE_AUTH_HEADER, - "expected".parse().expect("header should parse"), - ); - authorize_control_state_auth_header(&headers, "expected") - .expect("matching auth header should succeed"); -} - #[tokio::test] async fn local_controller_requests_require_internal_route_headers_not_framework_auth() { let (dir, state, state_path, _) = compile_framework_binding_control_state().await; @@ -7149,30 +7243,15 @@ async fn local_controller_requests_select_the_controller_for_the_current_site() assert_eq!(err.0.code, ProtocolErrorCode::Unauthorized); } -#[tokio::test] -async fn dynamic_caps_sidecar_requests_reject_ambient_control_state_auth_without_internal_route() { - let dir = TempDir::new().expect("temp dir"); - let state = compile_dynamic_caps_binding_state().await; - let state_path = dir.path().join("control-state.json"); - write_control_state(&state_path, &state).expect("state should write"); - let app = SiteControllerApp { - control: test_control_state_app(&dir, state, state_path), - ready: ready_site_controller_flag(), - }; - let mut headers = HeaderMap::new(); - headers.insert( - CONTROL_STATE_AUTH_HEADER, - app.control - .control_state_auth_token - .parse() - .expect("auth header should parse"), - ); +#[test] +fn dynamic_caps_sidecar_requests_require_internal_route_headers() { + let headers = HeaderMap::new(); let err = super::site_controller::authorize_dynamic_caps_sidecar_request( &headers, "components./alice", ) - .expect_err("ambient control-state auth must not bypass the controller internal route"); + .expect_err("dynamic-caps sidecar auth must require the router-injected internal route"); assert_eq!(err.0.code, ProtocolErrorCode::Unauthorized); } @@ -10110,7 +10189,6 @@ volumes: &temp.path().join("state"), &site_state_root, &artifact_root, - "test-auth", Some(SITE_CONTROLLER_MESH_IDENTITY_PATH), None, None, @@ -10391,7 +10469,6 @@ spec: &temp.path().join("state"), &site_state_root, &artifact_root, - "test-auth", Some(SITE_CONTROLLER_MESH_IDENTITY_PATH), None, None, From 295806cd1d38abb6c31cb1fe054fcd0db7a86700 Mon Sep 17 00:00:00 2001 From: Nick Hynes Date: Wed, 13 May 2026 13:28:05 +0800 Subject: [PATCH 25/25] Harden k8s rbac --- cli/tests/mixed_run/smoke.rs | 235 ++++++++++++++++++ .../site-controller/src/default_runtime.rs | 1 - .../default_runtime/kubernetes_controller.rs | 218 ++++++++++++++-- .../src/default_runtime/site_artifacts.rs | 145 +---------- runtime/site-controller/src/runtime_api.rs | 172 ++++++++++++- runtime/site-controller/src/tests.rs | 54 +++- 6 files changed, 652 insertions(+), 173 deletions(-) diff --git a/cli/tests/mixed_run/smoke.rs b/cli/tests/mixed_run/smoke.rs index f1aef60e..4c5812ca 100644 --- a/cli/tests/mixed_run/smoke.rs +++ b/cli/tests/mixed_run/smoke.rs @@ -911,6 +911,95 @@ fn wait_for_live_child(control_state_path: &Path, name: &str) -> u64 { .unwrap_or_else(|| panic!("dynamic child `{name}` should have an id")) } +fn rule_contains(rule: &Value, key: &str, expected: &str) -> bool { + rule.get(key) + .and_then(Value::as_array) + .is_some_and(|values| values.iter().any(|value| value.as_str() == Some(expected))) +} + +fn kubernetes_role(kind_state: &Value, name: &str) -> Value { + let namespace = kind_state["kubernetes_namespace"] + .as_str() + .expect("kubernetes site should publish namespace"); + let output = kubectl_for_manager_state(kind_state) + .arg("-n") + .arg(namespace) + .arg("get") + .arg("role") + .arg(name) + .arg("-o") + .arg("json") + .output() + .unwrap_or_else(|err| panic!("failed to query Kubernetes Role {name}: {err}")); + assert!( + output.status.success(), + "failed to query Kubernetes Role {name}:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + serde_json::from_slice(&output.stdout) + .unwrap_or_else(|err| panic!("Kubernetes Role {name} should parse: {err}")) +} + +fn assert_kubernetes_site_controller_role_cannot_write_rbac(kind_state: &Value) { + let role = kubernetes_role(kind_state, "amber-site-controller"); + let rules = role["rules"] + .as_array() + .expect("site-controller Role should include rules"); + assert!( + rules.iter().all(|rule| { + !rule_contains(rule, "apiGroups", "rbac.authorization.k8s.io") + && !rule_contains(rule, "resources", "serviceaccounts") + && !rule_contains(rule, "resources", "roles") + && !rule_contains(rule, "resources", "rolebindings") + }), + "site-controller Role should not be able to create or update Kubernetes RBAC objects" + ); +} + +fn assert_kubernetes_dynamic_provisioner_role_is_secret_only(kind_state: &Value) { + let role = kubernetes_role(kind_state, "amber-provisioner-dynamic"); + let rules = role["rules"] + .as_array() + .expect("dynamic provisioner Role should include rules"); + assert_eq!( + rules.len(), + 1, + "dynamic provisioner Role should contain only the mesh-secret write rule" + ); + let rule = &rules[0]; + assert_string_array_members( + &rule["resources"], + &["secrets"], + "dynamic provisioner Role should be limited to Kubernetes Secrets", + ); + assert_string_array_members( + &rule["verbs"], + &["create", "get", "update"], + "dynamic provisioner Role should only cover the verbs required for generated secrets", + ); +} + +fn assert_kubernetes_dynamic_child_artifact_omits_provisioner_rbac(artifact: &Path) { + let kustomization = fs::read_to_string(artifact.join("kustomization.yaml")) + .expect("kubernetes dynamic child kustomization should read"); + assert!(kustomization.contains("01-configmaps/amber-mesh-provision.yaml")); + assert!(kustomization.contains("02-rbac/amber-provisioner-job.yaml")); + assert!(!kustomization.contains("02-rbac/amber-provisioner-sa.yaml")); + assert!(!kustomization.contains("02-rbac/amber-provisioner-role.yaml")); + assert!(!kustomization.contains("02-rbac/amber-provisioner-rolebinding.yaml")); + assert!(!artifact.join("02-rbac/amber-provisioner-sa.yaml").exists()); + assert!( + !artifact + .join("02-rbac/amber-provisioner-role.yaml") + .exists() + ); + assert!( + !artifact + .join("02-rbac/amber-provisioner-rolebinding.yaml") + .exists() + ); +} + fn framework_child_artifact(run: &RunHandle, site_id: &str, child_id: u64) -> PathBuf { framework_child_artifact_dir(&run.run_root, site_id, child_id) } @@ -1108,6 +1197,67 @@ fn write_framework_dynamic_child_manifest( ); } +fn write_framework_kubernetes_dynamic_child_fixture( + root: &Path, + kind_cluster: &KindCluster, +) -> ScenarioFixture { + fs::write(root.join("admin.py"), FRAMEWORK_ADMIN_APP).expect("failed to write admin.py"); + fs::write(root.join("worker.py"), FRAMEWORK_WORKER_APP).expect("failed to write worker.py"); + write_framework_admin_component(root, "admin.json5", true, 8080); + write_framework_worker_component(root, "child-kind-root.json5", true, "child-kind-root", 8080); + write_framework_dynamic_child_manifest(root, "child-kind.json5", "child-kind-root.json5", &[]); + + let manifest = root.join("root.json5"); + write_json( + &manifest, + &json!({ + "manifest_version": "0.3.0", + "slots": { + "realm": { "kind": "component", "optional": true } + }, + "components": { + "admin": "./admin.json5" + }, + "child_templates": { + "child_kind": { "manifest": "./child-kind.json5" } + }, + "bindings": [ + { "to": "#admin.ctl", "from": "framework.component" } + ], + "exports": { + "admin_http": "#admin.http" + } + }), + ); + + let placement = root.join("placement.json5"); + write_json( + &placement, + &json!({ + "schema": "amber.run.placement", + "version": 1, + "sites": { + "kind_local": { + "kind": "kubernetes", + "context": kind_cluster.context_name() + } + }, + "defaults": { + "image": "kind_local" + }, + "components": { + "/admin": "kind_local", + "/job-kind/root": "kind_local" + } + }), + ); + + ScenarioFixture { + manifest, + placement, + } +} + fn write_framework_matrix_fixture(root: &Path, kind_cluster: &KindCluster) -> ScenarioFixture { fs::write(root.join("admin.py"), FRAMEWORK_ADMIN_APP).expect("failed to write admin.py"); fs::write(root.join("app.py"), FRAMEWORK_MATRIX_APP).expect("failed to write app.py"); @@ -3789,6 +3939,91 @@ fn framework_component_destroy_of_provider_keeps_consumer_live() { run.stop(); } +#[test] +#[ignore = "requires docker + kind + kubectl + qemu + an Ubuntu 24.04 cloud image matching the \ + host architecture; run manually or in CI"] +fn framework_component_kubernetes_dynamic_child_rbac_live() { + ensure_internal_images(); + let temp = temp_output_dir("framework-component-kubernetes-rbac-"); + let kubeconfig = temp.path().join("kubeconfig"); + let kind_cluster = KindCluster::from_env_or_create(&kubeconfig); + ensure_kind_internal_images(&kind_cluster); + let kubeconfig_env = kind_cluster.kubeconfig.display().to_string(); + + let fixture = write_framework_kubernetes_dynamic_child_fixture(temp.path(), &kind_cluster); + let storage_root = temp.path().join("state"); + let mut run = run_manifest_with_env( + &fixture.manifest, + &fixture.placement, + &storage_root, + &[("KUBECONFIG", &kubeconfig_env)], + ); + + let kind_state = wait_for_state_status( + &run.run_root, + "kind_local", + "running", + Duration::from_secs(120), + ); + assert_kubernetes_site_controller_role_cannot_write_rbac(&kind_state); + assert_kubernetes_dynamic_provisioner_role_is_secret_only(&kind_state); + + let control_state_path = framework_control_state_path(&run); + let creator_port = pick_free_port(); + let mut creator_proxy = spawn_framework_proxy_for_site( + &run.site_artifact_dir("kind_local"), + "admin_http", + creator_port, + &kind_state, + ); + wait_for_path( + &mut creator_proxy, + creator_port, + "/id", + framework_mutation_request_timeout(), + ); + assert_eq!( + wait_for_body( + &mut creator_proxy, + creator_port, + "/id", + Duration::from_secs(30), + ), + "admin" + ); + + let (create_status, create_response) = framework_admin_get( + creator_port, + "/create/child_kind/job-kind", + "create request should return an HTTP response", + ); + assert_eq!( + create_status, 200, + "create request should succeed; response: {create_response}" + ); + let child_id = wait_for_live_child(&control_state_path, "job-kind"); + + let root_artifact = framework_child_artifact(&run, "kind_local", child_id); + assert_kubernetes_dynamic_child_artifact_omits_provisioner_rbac(&root_artifact); + let root_port = pick_free_port(); + let mut root_proxy = + spawn_framework_proxy_for_site(&root_artifact, "http", root_port, &kind_state); + wait_for_path( + &mut root_proxy, + root_port, + "/id", + framework_mutation_request_timeout(), + ); + assert_eq!( + wait_for_body(&mut root_proxy, root_port, "/id", Duration::from_secs(30)), + "child-kind-root" + ); + + stop_proxy(&mut root_proxy); + stop_proxy(&mut creator_proxy); + run.stop(); +} + #[test] #[ignore = "requires docker + kind + kubectl + qemu + an Ubuntu 24.04 cloud image matching the \ host architecture; run manually or in CI"] diff --git a/runtime/site-controller/src/default_runtime.rs b/runtime/site-controller/src/default_runtime.rs index a67f9dff..6c11f41c 100644 --- a/runtime/site-controller/src/default_runtime.rs +++ b/runtime/site-controller/src/default_runtime.rs @@ -98,7 +98,6 @@ const COMPONENT_CONTROL_SOCKET_PATH_IN_VOLUME: &str = "/router-control.sock"; const COMPOSE_PROVISIONER_SERVICE_NAME: &str = "amber-provisioner"; const COMPOSE_ROUTER_SERVICE_NAME: &str = "amber-router"; const KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH: &str = "01-configmaps/amber-mesh-provision.yaml"; -const KUBERNETES_PROVISIONER_JOB_PATH: &str = "02-rbac/amber-provisioner-job.yaml"; const KUBERNETES_PROVISIONER_ROLE_PATH: &str = "02-rbac/amber-provisioner-role.yaml"; const KUBERNETES_PROVISIONER_ROLEBINDING_PATH: &str = "02-rbac/amber-provisioner-rolebinding.yaml"; const KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH: &str = "02-rbac/amber-provisioner-sa.yaml"; diff --git a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs index 88a6b296..0fe2a30d 100644 --- a/runtime/site-controller/src/default_runtime/kubernetes_controller.rs +++ b/runtime/site-controller/src/default_runtime/kubernetes_controller.rs @@ -40,6 +40,12 @@ const KUBERNETES_CONTROLLER_PLAN_PATH: &str = "/amber/site/state/site-controller const KUBERNETES_CONTROLLER_STATE_PATH: &str = "/amber/site/state/site-controller-state.json"; const KUBERNETES_CONTROLLER_DESIRED_LINKS_PATH: &str = "/amber/site/state/desired-links.json"; const KUBERNETES_CONTROLLER_SEED_ROOT: &str = "/amber/seed"; +const KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_NAME: &str = "amber-provisioner"; +const KUBERNETES_DYNAMIC_PROVISIONER_ROLE_NAME: &str = "amber-provisioner-dynamic"; +const KUBERNETES_DYNAMIC_PROVISIONER_ROLE_PATH: &str = + "02-rbac/amber-provisioner-dynamic-role.yaml"; +const KUBERNETES_DYNAMIC_PROVISIONER_ROLE_BINDING_PATH: &str = + "02-rbac/amber-provisioner-dynamic-rolebinding.yaml"; #[derive(Clone, Debug)] struct KubernetesControllerSeedSecret { @@ -92,6 +98,7 @@ pub fn inject_kubernetes_site_controller( } }), )?; + write_kubernetes_dynamic_provisioner_rbac(artifact_root)?; write_yaml_artifact( artifact_root.join(KUBERNETES_CONTROLLER_ROLE_PATH), &json!({ @@ -108,7 +115,6 @@ pub fn inject_kubernetes_site_controller( "configmaps", "persistentvolumeclaims", "secrets", - "serviceaccounts", "services" ], "verbs": ["create", "delete", "get", "list", "patch", "update", "watch"], @@ -127,11 +133,6 @@ pub fn inject_kubernetes_site_controller( "apiGroups": ["networking.k8s.io"], "resources": ["networkpolicies"], "verbs": ["create", "delete", "get", "list", "patch", "update", "watch"], - }, - { - "apiGroups": ["rbac.authorization.k8s.io"], - "resources": ["roles", "rolebindings"], - "verbs": ["create", "delete", "get", "list", "patch", "update", "watch"], } ] }), @@ -168,6 +169,9 @@ pub fn inject_kubernetes_site_controller( .collect::>(); resource_paths.extend([ KUBERNETES_CONTROLLER_STATE_PVC_PATH, + KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH, + KUBERNETES_DYNAMIC_PROVISIONER_ROLE_PATH, + KUBERNETES_DYNAMIC_PROVISIONER_ROLE_BINDING_PATH, KUBERNETES_CONTROLLER_SERVICE_ACCOUNT_PATH, KUBERNETES_CONTROLLER_ROLE_PATH, KUBERNETES_CONTROLLER_ROLE_BINDING_PATH, @@ -549,6 +553,57 @@ fn render_kubernetes_controller_seed_secret( }) } +fn write_kubernetes_dynamic_provisioner_rbac(artifact_root: &Path) -> Result<()> { + let labels = kubernetes_provisioner_labels(); + write_yaml_artifact_if_missing( + artifact_root.join(KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH), + &json!({ + "apiVersion": "v1", + "kind": "ServiceAccount", + "metadata": { + "name": KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_NAME, + "labels": labels, + } + }), + )?; + write_yaml_artifact( + artifact_root.join(KUBERNETES_DYNAMIC_PROVISIONER_ROLE_PATH), + &json!({ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "Role", + "metadata": { + "name": KUBERNETES_DYNAMIC_PROVISIONER_ROLE_NAME, + "labels": kubernetes_provisioner_labels(), + }, + "rules": [{ + "apiGroups": [""], + "resources": ["secrets"], + "verbs": ["create", "get", "update"], + }] + }), + )?; + write_yaml_artifact( + artifact_root.join(KUBERNETES_DYNAMIC_PROVISIONER_ROLE_BINDING_PATH), + &json!({ + "apiVersion": "rbac.authorization.k8s.io/v1", + "kind": "RoleBinding", + "metadata": { + "name": KUBERNETES_DYNAMIC_PROVISIONER_ROLE_NAME, + "labels": kubernetes_provisioner_labels(), + }, + "subjects": [{ + "kind": "ServiceAccount", + "name": KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_NAME, + }], + "roleRef": { + "apiGroup": "rbac.authorization.k8s.io", + "kind": "Role", + "name": KUBERNETES_DYNAMIC_PROVISIONER_ROLE_NAME, + } + }), + ) +} + fn add_kubernetes_resource_paths(artifact_root: &Path, paths: &[&str]) -> Result<()> { let kustomization_path = artifact_root.join("kustomization.yaml"); let raw = fs::read_to_string(&kustomization_path) @@ -682,7 +737,8 @@ fn ensure_kubernetes_router_allows_site_controller_ingress( let controller_component = controller_selector .get("amber.io/component") .expect("selector must contain component"); - let mut required_ports = BTreeSet::from([KUBERNETES_ROUTER_CONTROL_PORT]); + let mut required_ports = + BTreeSet::from([KUBERNETES_ROUTER_CONTROL_PORT, KUBERNETES_ROUTER_MESH_PORT]); required_ports.extend(route_ports.iter().copied()); if let Some(rule) = ingress.iter_mut().find(|rule| { rule.as_mapping() @@ -763,9 +819,9 @@ fn ensure_kubernetes_site_controller_allows_router_egress( artifact_root: &Path, route_ports: &BTreeSet, ) -> Result<()> { - if route_ports.is_empty() { - return Ok(()); - } + let mut required_ports = route_ports.clone(); + required_ports.insert(KUBERNETES_ROUTER_CONTROL_PORT); + required_ports.insert(KUBERNETES_ROUTER_MESH_PORT); let path = artifact_root.join(KUBERNETES_CONTROLLER_NETPOL_PATH); let raw = fs::read_to_string(&path) .into_diagnostic() @@ -826,7 +882,7 @@ fn ensure_kubernetes_site_controller_allows_router_egress( .and_then(|port| u16::try_from(port).ok()) }) .collect::>(); - for port in route_ports { + for port in &required_ports { if existing_ports.contains(port) { continue; } @@ -847,7 +903,7 @@ fn ensure_kubernetes_site_controller_allows_router_egress( "matchLabels": router_selector, } }], - "ports": route_ports.iter().map(|port| json!({ + "ports": required_ports.iter().map(|port| json!({ "protocol": "TCP", "port": port, })).collect::>() @@ -878,6 +934,20 @@ fn kubernetes_controller_labels() -> BTreeMap { ]) } +fn kubernetes_provisioner_labels() -> BTreeMap { + BTreeMap::from([ + ( + "app.kubernetes.io/managed-by".to_string(), + "amber".to_string(), + ), + ( + "amber.io/component".to_string(), + KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_NAME.to_string(), + ), + ("amber.io/type".to_string(), "provisioner".to_string()), + ]) +} + fn kubernetes_controller_selector() -> BTreeMap { BTreeMap::from([( "amber.io/component".to_string(), @@ -903,6 +973,13 @@ fn write_yaml_artifact(path: PathBuf, value: &impl serde::Serialize) -> Result<( .wrap_err_with(|| format!("failed to write {}", path.display())) } +fn write_yaml_artifact_if_missing(path: PathBuf, value: &impl serde::Serialize) -> Result<()> { + if path.exists() { + return Ok(()); + } + write_yaml_artifact(path, value) +} + #[cfg(test)] mod tests { use std::{collections::BTreeMap, net::SocketAddr, path::PathBuf}; @@ -989,8 +1066,6 @@ spec: matchLabels: amber.io/component: amber-router ports: - - protocol: TCP - port: 24000 - protocol: TCP port: 24100 "#, @@ -1067,6 +1142,16 @@ spec: .expect("controller deployment should include a pod spec") } + fn rule_values<'a>(rule: &'a serde_yaml::Value, key: &str) -> Vec<&'a str> { + rule.as_mapping() + .and_then(|mapping| mapping.get(yaml_string(key))) + .and_then(serde_yaml::Value::as_sequence) + .expect("rule sequence should exist") + .iter() + .map(|value| value.as_str().expect("rule value should be a string")) + .collect() + } + #[test] fn inject_kubernetes_site_controller_propagates_launch_env() { let temp = tempfile::tempdir().expect("temp dir"); @@ -1233,6 +1318,111 @@ spec: ); } + #[test] + fn inject_kubernetes_site_controller_bootstraps_dynamic_provisioner_rbac() { + let temp = tempfile::tempdir().expect("temp dir"); + let artifact_root = temp.path(); + write_kubernetes_controller_fixture(artifact_root); + + let plan = test_plan(artifact_root); + write_controller_state(&plan); + + inject_kubernetes_site_controller( + artifact_root, + &plan, + "ghcr.io/rdi-foundation/amber-site-controller:test", + ) + .expect("kubernetes site controller injection should succeed"); + + let kustomization = + fs::read_to_string(artifact_root.join("kustomization.yaml")).expect("kustomization"); + assert!(kustomization.contains(KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH)); + assert!(kustomization.contains(KUBERNETES_DYNAMIC_PROVISIONER_ROLE_PATH)); + assert!(kustomization.contains(KUBERNETES_DYNAMIC_PROVISIONER_ROLE_BINDING_PATH)); + + let controller_role_raw = + fs::read_to_string(artifact_root.join(KUBERNETES_CONTROLLER_ROLE_PATH)) + .expect("controller role should exist"); + let controller_role: serde_yaml::Value = + serde_yaml::from_str(&controller_role_raw).expect("controller role should parse"); + let controller_rules = controller_role["rules"] + .as_sequence() + .expect("controller role should have rules"); + assert!( + controller_rules.iter().all(|rule| { + !rule_values(rule, "apiGroups").contains(&"rbac.authorization.k8s.io") + && !rule_values(rule, "resources").contains(&"serviceaccounts") + && !rule_values(rule, "resources").contains(&"roles") + && !rule_values(rule, "resources").contains(&"rolebindings") + }), + "site-controller Role must not be able to rewrite Kubernetes RBAC objects" + ); + + let provisioner_role_raw = + fs::read_to_string(artifact_root.join(KUBERNETES_DYNAMIC_PROVISIONER_ROLE_PATH)) + .expect("dynamic provisioner role should exist"); + let provisioner_role: serde_yaml::Value = + serde_yaml::from_str(&provisioner_role_raw).expect("provisioner role should parse"); + assert_eq!( + provisioner_role["metadata"]["name"].as_str(), + Some(KUBERNETES_DYNAMIC_PROVISIONER_ROLE_NAME) + ); + let provisioner_rules = provisioner_role["rules"] + .as_sequence() + .expect("provisioner role should have rules"); + assert_eq!(provisioner_rules.len(), 1); + let rule = &provisioner_rules[0]; + assert_eq!(rule_values(rule, "apiGroups"), vec![""]); + assert_eq!(rule_values(rule, "resources"), vec!["secrets"]); + let verbs = rule_values(rule, "verbs"); + assert!(verbs.contains(&"create")); + assert!(verbs.contains(&"get")); + assert!(verbs.contains(&"update")); + assert!( + !rule + .as_mapping() + .expect("provisioner rule should be a mapping") + .contains_key(yaml_string("resourceNames")), + "dynamic provisioner secret names are not all known at site bootstrap time" + ); + + let role_binding_raw = fs::read_to_string( + artifact_root.join(KUBERNETES_DYNAMIC_PROVISIONER_ROLE_BINDING_PATH), + ) + .expect("dynamic provisioner rolebinding should exist"); + let role_binding: serde_yaml::Value = + serde_yaml::from_str(&role_binding_raw).expect("rolebinding should parse"); + assert_eq!( + role_binding["subjects"][0]["name"].as_str(), + Some(KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_NAME) + ); + assert_eq!( + role_binding["roleRef"]["name"].as_str(), + Some(KUBERNETES_DYNAMIC_PROVISIONER_ROLE_NAME) + ); + + let router_netpol_raw = + fs::read_to_string(artifact_root.join(KUBERNETES_ROUTER_NETPOL_PATH)) + .expect("router network policy should exist"); + let router_netpol: serde_yaml::Value = + serde_yaml::from_str(&router_netpol_raw).expect("router network policy should parse"); + let router_ingress_ports = router_netpol["spec"]["ingress"] + .as_sequence() + .expect("router network policy should include ingress") + .iter() + .flat_map(|rule| rule["ports"].as_sequence().into_iter().flatten()) + .filter_map(|port| port["port"].as_u64()) + .collect::>(); + assert!( + router_ingress_ports.contains(&u64::from(KUBERNETES_ROUTER_MESH_PORT)), + "site-controller must be allowed to probe the router mesh listener" + ); + assert!( + router_ingress_ports.contains(&u64::from(KUBERNETES_ROUTER_CONTROL_PORT)), + "site-controller must be allowed to probe the router control listener" + ); + } + #[test] fn inject_kubernetes_site_controller_chunks_seed_secrets_to_avoid_object_limits() { let temp = tempfile::tempdir().expect("temp dir"); diff --git a/runtime/site-controller/src/default_runtime/site_artifacts.rs b/runtime/site-controller/src/default_runtime/site_artifacts.rs index 0f5f203e..8aea93a7 100644 --- a/runtime/site-controller/src/default_runtime/site_artifacts.rs +++ b/runtime/site-controller/src/default_runtime/site_artifacts.rs @@ -1765,148 +1765,6 @@ pub fn set_site_artifact_mesh_identity_seed( )) } -fn kubernetes_resource_name(document: &serde_yaml::Value) -> Option<&str> { - document - .as_mapping() - .and_then(|root| root.get(yaml_string("metadata"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|metadata| metadata.get(yaml_string("name"))) - .and_then(serde_yaml::Value::as_str) -} - -fn kubernetes_dynamic_apply_resource_kept_from_contents( - resource: &str, - raw: &str, - child_component_labels: &BTreeSet, -) -> Result { - if matches!( - resource, - KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH - | KUBERNETES_PROVISIONER_JOB_PATH - | KUBERNETES_PROVISIONER_ROLE_PATH - | KUBERNETES_PROVISIONER_ROLEBINDING_PATH - | KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH - ) || resource.starts_with("03-persistentvolumeclaims/") - { - return Ok(true); - } - - let document: serde_yaml::Value = serde_yaml::from_str(raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid kubernetes resource {resource}"))?; - Ok(document - .as_mapping() - .and_then(|root| root.get(yaml_string("metadata"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|metadata| metadata.get(yaml_string("labels"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|labels| labels.get(yaml_string("amber.io/component-id"))) - .and_then(serde_yaml::Value::as_str) - .is_some_and(|component_id| child_component_labels.contains(component_id))) -} - -pub(crate) fn project_kubernetes_dynamic_child_artifact_files( - artifact_files: &BTreeMap, - component_ids: &[usize], -) -> Result> { - let child_component_labels = component_ids - .iter() - .map(|component_id| format!("c{component_id}")) - .collect::>(); - let kustomization_path = "kustomization.yaml"; - let raw = artifact_files.get(kustomization_path).ok_or_else(|| { - miette::miette!("dynamic kubernetes artifact snapshot is missing {kustomization_path}") - })?; - let mut document: serde_yaml::Value = serde_yaml::from_str(raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid kustomization {kustomization_path}"))?; - let root = document.as_mapping_mut().ok_or_else(|| { - miette::miette!("kustomization {kustomization_path} is not a YAML mapping") - })?; - let resources = root - .get_mut(yaml_string("resources")) - .and_then(serde_yaml::Value::as_sequence_mut) - .ok_or_else(|| { - miette::miette!("kustomization {kustomization_path} is missing a resources sequence") - })?; - let mut projected = artifact_files - .iter() - .filter(|(path, _)| !path.ends_with(".yaml") && path.as_str() != kustomization_path) - .map(|(path, contents)| (path.clone(), contents.clone())) - .collect::>(); - let mut kept_resources = Vec::new(); - let mut kept_resource_names = BTreeSet::new(); - for resource in resources - .iter() - .filter_map(serde_yaml::Value::as_str) - .map(str::to_owned) - { - let raw = artifact_files - .get(&resource) - .ok_or_else(|| miette::miette!("dynamic kubernetes artifact is missing {resource}"))?; - if !kubernetes_dynamic_apply_resource_kept_from_contents( - &resource, - raw, - &child_component_labels, - )? { - continue; - } - let document: serde_yaml::Value = serde_yaml::from_str(raw) - .into_diagnostic() - .wrap_err_with(|| format!("invalid kubernetes resource {resource}"))?; - if let Some(name) = kubernetes_resource_name(&document) { - kept_resource_names.insert(name.to_string()); - } - projected.insert(resource.clone(), raw.clone()); - kept_resources.push(serde_yaml::Value::String(resource)); - } - *resources = kept_resources; - - if let Some(generators) = root - .get_mut(yaml_string("secretGenerator")) - .and_then(serde_yaml::Value::as_sequence_mut) - { - generators.retain(|generator| { - generator - .as_mapping() - .and_then(|mapping| mapping.get(yaml_string("name"))) - .and_then(serde_yaml::Value::as_str) - != Some(KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME) - }); - } - - if let Some(replacements) = root - .get_mut(yaml_string("replacements")) - .and_then(serde_yaml::Value::as_sequence_mut) - { - replacements.retain_mut(|replacement| { - let Some(targets) = replacement - .as_mapping_mut() - .and_then(|mapping| mapping.get_mut(yaml_string("targets"))) - .and_then(serde_yaml::Value::as_sequence_mut) - else { - return false; - }; - targets.retain(|target| { - target - .as_mapping() - .and_then(|mapping| mapping.get(yaml_string("select"))) - .and_then(serde_yaml::Value::as_mapping) - .and_then(|select| select.get(yaml_string("name"))) - .and_then(serde_yaml::Value::as_str) - .is_some_and(|name| kept_resource_names.contains(name)) - }); - !targets.is_empty() - }); - } - - let rendered = serde_yaml::to_string(&document) - .into_diagnostic() - .wrap_err_with(|| format!("failed to serialize {kustomization_path}"))?; - projected.insert(kustomization_path.to_string(), rendered); - Ok(projected) -} - pub(super) fn project_kubernetes_dynamic_child_destroy_artifact_files( artifact_files: &BTreeMap, ) -> Result> { @@ -2005,7 +1863,8 @@ fn rewrite_dynamic_kubernetes_apply_bundle( component_ids: &[usize], ) -> Result<()> { let files = read_artifact_snapshot(artifact_root)?; - let projected = project_kubernetes_dynamic_child_artifact_files(&files, component_ids)?; + let projected = + crate::runtime_api::project_kubernetes_dynamic_child_artifact_files(&files, component_ids)?; replace_artifact_snapshot(artifact_root, &projected) } diff --git a/runtime/site-controller/src/runtime_api.rs b/runtime/site-controller/src/runtime_api.rs index a91c74df..7b24bb96 100644 --- a/runtime/site-controller/src/runtime_api.rs +++ b/runtime/site-controller/src/runtime_api.rs @@ -1,5 +1,5 @@ use std::{ - collections::BTreeMap, + collections::{BTreeMap, BTreeSet}, future::Future, net::SocketAddr, path::{Path, PathBuf}, @@ -18,9 +18,6 @@ use super::state::{FrameworkControlState, LiveChildRecord, SiteControllerPlan}; const KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH: &str = "01-configmaps/amber-mesh-provision.yaml"; const KUBERNETES_PROVISIONER_JOB_PATH: &str = "02-rbac/amber-provisioner-job.yaml"; -const KUBERNETES_PROVISIONER_ROLE_PATH: &str = "02-rbac/amber-provisioner-role.yaml"; -const KUBERNETES_PROVISIONER_ROLEBINDING_PATH: &str = "02-rbac/amber-provisioner-rolebinding.yaml"; -const KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH: &str = "02-rbac/amber-provisioner-sa.yaml"; const KUBERNETES_ROUTER_EXTERNAL_SECRET_NAME: &str = "amber-router-external"; const SITE_CONTROLLER_RUNTIME_PLAN_SCHEMA: &str = "amber.run.site_controller_runtime_plan"; const SITE_CONTROLLER_RUNTIME_PLAN_VERSION: u32 = 1; @@ -294,15 +291,11 @@ fn kubernetes_resource_name(document: &serde_yaml::Value) -> Option<&str> { fn kubernetes_dynamic_apply_resource_kept_from_contents( resource: &str, raw: &str, - child_component_labels: &std::collections::BTreeSet, + child_component_labels: &BTreeSet, ) -> Result { if matches!( resource, - KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH - | KUBERNETES_PROVISIONER_JOB_PATH - | KUBERNETES_PROVISIONER_ROLE_PATH - | KUBERNETES_PROVISIONER_ROLEBINDING_PATH - | KUBERNETES_PROVISIONER_SERVICE_ACCOUNT_PATH + KUBERNETES_MESH_PROVISION_CONFIGMAP_PATH | KUBERNETES_PROVISIONER_JOB_PATH ) || resource.starts_with("03-persistentvolumeclaims/") { return Ok(true); @@ -329,7 +322,7 @@ pub fn project_kubernetes_dynamic_child_artifact_files( let child_component_labels = component_ids .iter() .map(|component_id| format!("c{component_id}")) - .collect::>(); + .collect::>(); let kustomization_path = "kustomization.yaml"; let raw = artifact_files.get(kustomization_path).ok_or_else(|| { miette::miette!("dynamic kubernetes artifact snapshot is missing {kustomization_path}") @@ -352,7 +345,7 @@ pub fn project_kubernetes_dynamic_child_artifact_files( .map(|(path, contents)| (path.clone(), contents.clone())) .collect::>(); let mut kept_resources = Vec::new(); - let mut kept_resource_names = std::collections::BTreeSet::new(); + let mut kept_resource_names = BTreeSet::new(); for resource in resources .iter() .filter_map(serde_yaml::Value::as_str) @@ -498,3 +491,158 @@ pub fn published_router_mesh_addr_for_consumer_kind( SiteKind::Direct | SiteKind::Vm => None, } } + +#[cfg(test)] +mod tests { + use super::*; + + fn yaml_sequence_strings<'a>(document: &'a serde_yaml::Value, key: &str) -> Vec<&'a str> { + document + .as_mapping() + .and_then(|root| root.get(yaml_string(key))) + .and_then(serde_yaml::Value::as_sequence) + .expect("sequence should exist") + .iter() + .map(|value| value.as_str().expect("sequence entry should be a string")) + .collect() + } + + #[test] + fn dynamic_kubernetes_child_projection_drops_provisioner_rbac() { + let files = BTreeMap::from([ + ( + "kustomization.yaml".to_string(), + r#" +resources: + - 01-configmaps/amber-mesh-provision.yaml + - 02-rbac/amber-provisioner-sa.yaml + - 02-rbac/amber-provisioner-role.yaml + - 02-rbac/amber-provisioner-rolebinding.yaml + - 02-rbac/amber-provisioner-job.yaml + - 03-persistentvolumeclaims/child-state.yaml + - 03-deployments/child.yaml + - 03-deployments/other.yaml +secretGenerator: + - name: amber-router-external + - name: child-secret +replacements: + - source: + kind: ConfigMap + name: amber-mesh-provision + targets: + - select: + name: child + - select: + name: other +"# + .to_string(), + ), + ( + "01-configmaps/amber-mesh-provision.yaml".to_string(), + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: amber-mesh-provision\n" + .to_string(), + ), + ( + "02-rbac/amber-provisioner-sa.yaml".to_string(), + "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: amber-provisioner\n" + .to_string(), + ), + ( + "02-rbac/amber-provisioner-role.yaml".to_string(), + "apiVersion: rbac.authorization.k8s.io/v1\nkind: Role\nmetadata:\n name: \ + amber-provisioner\n" + .to_string(), + ), + ( + "02-rbac/amber-provisioner-rolebinding.yaml".to_string(), + "apiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: \ + amber-provisioner\n" + .to_string(), + ), + ( + "02-rbac/amber-provisioner-job.yaml".to_string(), + "apiVersion: batch/v1\nkind: Job\nmetadata:\n name: amber-provisioner\n" + .to_string(), + ), + ( + "03-persistentvolumeclaims/child-state.yaml".to_string(), + "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: child-state\n" + .to_string(), + ), + ( + "03-deployments/child.yaml".to_string(), + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: child\n labels:\n \ + amber.io/component-id: c1\n" + .to_string(), + ), + ( + "03-deployments/other.yaml".to_string(), + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: other\n labels:\n \ + amber.io/component-id: c2\n" + .to_string(), + ), + ("README.md".to_string(), "operator notes".to_string()), + ]); + + let projected = project_kubernetes_dynamic_child_artifact_files(&files, &[1]) + .expect("dynamic child projection should succeed"); + + assert!(projected.contains_key("01-configmaps/amber-mesh-provision.yaml")); + assert!(projected.contains_key("02-rbac/amber-provisioner-job.yaml")); + assert!(projected.contains_key("03-persistentvolumeclaims/child-state.yaml")); + assert!(projected.contains_key("03-deployments/child.yaml")); + assert_eq!( + projected.get("README.md").map(String::as_str), + Some("operator notes") + ); + assert!(!projected.contains_key("02-rbac/amber-provisioner-sa.yaml")); + assert!(!projected.contains_key("02-rbac/amber-provisioner-role.yaml")); + assert!(!projected.contains_key("02-rbac/amber-provisioner-rolebinding.yaml")); + assert!(!projected.contains_key("03-deployments/other.yaml")); + + let kustomization: serde_yaml::Value = + serde_yaml::from_str(&projected["kustomization.yaml"]) + .expect("projected kustomization should parse"); + assert_eq!( + yaml_sequence_strings(&kustomization, "resources"), + vec![ + "01-configmaps/amber-mesh-provision.yaml", + "02-rbac/amber-provisioner-job.yaml", + "03-persistentvolumeclaims/child-state.yaml", + "03-deployments/child.yaml", + ] + ); + let generators = kustomization + .as_mapping() + .and_then(|root| root.get(yaml_string("secretGenerator"))) + .and_then(serde_yaml::Value::as_sequence) + .expect("secret generators should remain"); + assert_eq!(generators.len(), 1); + assert_eq!( + generators[0] + .as_mapping() + .and_then(|mapping| mapping.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str), + Some("child-secret") + ); + let replacement_targets = kustomization + .as_mapping() + .and_then(|root| root.get(yaml_string("replacements"))) + .and_then(serde_yaml::Value::as_sequence) + .and_then(|replacements| replacements.first()) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|replacement| replacement.get(yaml_string("targets"))) + .and_then(serde_yaml::Value::as_sequence) + .expect("kept replacement should have targets"); + assert_eq!(replacement_targets.len(), 1); + assert_eq!( + replacement_targets[0] + .as_mapping() + .and_then(|target| target.get(yaml_string("select"))) + .and_then(serde_yaml::Value::as_mapping) + .and_then(|select| select.get(yaml_string("name"))) + .and_then(serde_yaml::Value::as_str), + Some("child") + ); + } +} diff --git a/runtime/site-controller/src/tests.rs b/runtime/site-controller/src/tests.rs index 2a853d6c..448c5652 100644 --- a/runtime/site-controller/src/tests.rs +++ b/runtime/site-controller/src/tests.rs @@ -10289,7 +10289,7 @@ volumes: assert!( volumes .iter() - .any(|value| { value.as_str() == Some("/var/run/docker.sock:/var/run/docker.sock") }) + .any(|value| value.as_str() == Some("/var/run/docker.sock:/var/run/docker.sock")) ); let group_add = service .get(serde_yaml::Value::String("group_add".to_string())) @@ -10383,8 +10383,6 @@ spec: matchLabels: amber.io/component: amber-router ports: - - protocol: TCP - port: 24000 - protocol: TCP port: 24100 "#, @@ -10492,9 +10490,53 @@ spec: .expect("kustomization should read"); assert!(kustomization.contains("01-secrets/amber-site-controller-seed.yaml")); assert!(kustomization.contains("03-persistentvolumeclaims/amber-site-controller-state.yaml")); + assert!(kustomization.contains("02-rbac/amber-provisioner-sa.yaml")); + assert!(kustomization.contains("02-rbac/amber-provisioner-dynamic-role.yaml")); + assert!(kustomization.contains("02-rbac/amber-provisioner-dynamic-rolebinding.yaml")); assert!(kustomization.contains("03-deployments/amber-site-controller.yaml")); assert!(kustomization.contains("04-services/amber-site-controller.yaml")); + fn rule_contains(rule: &serde_yaml::Value, key: &str, expected: &str) -> bool { + rule.as_mapping() + .and_then(|mapping| mapping.get(serde_yaml::Value::String(key.to_string()))) + .and_then(serde_yaml::Value::as_sequence) + .is_some_and(|values| values.iter().any(|value| value.as_str() == Some(expected))) + } + + let controller_role_raw = + fs::read_to_string(artifact_root.join("02-rbac/amber-site-controller-role.yaml")) + .expect("controller role should read"); + let controller_role: serde_yaml::Value = + serde_yaml::from_str(&controller_role_raw).expect("controller role should parse"); + let controller_role_rules = controller_role["rules"] + .as_sequence() + .expect("controller role should include rules"); + assert!( + controller_role_rules.iter().all(|rule| { + !rule_contains(rule, "apiGroups", "rbac.authorization.k8s.io") + && !rule_contains(rule, "resources", "serviceaccounts") + && !rule_contains(rule, "resources", "roles") + && !rule_contains(rule, "resources", "rolebindings") + }), + "site-controller Role should not be able to create or update Kubernetes RBAC objects" + ); + + let dynamic_provisioner_role_raw = + fs::read_to_string(artifact_root.join("02-rbac/amber-provisioner-dynamic-role.yaml")) + .expect("dynamic provisioner role should read"); + let dynamic_provisioner_role: serde_yaml::Value = + serde_yaml::from_str(&dynamic_provisioner_role_raw) + .expect("dynamic provisioner role should parse"); + let provisioner_rules = dynamic_provisioner_role["rules"] + .as_sequence() + .expect("dynamic provisioner role should include rules"); + assert_eq!(provisioner_rules.len(), 1); + let rule = &provisioner_rules[0]; + assert!(rule_contains(rule, "resources", "secrets")); + assert!(rule_contains(rule, "verbs", "create")); + assert!(rule_contains(rule, "verbs", "get")); + assert!(rule_contains(rule, "verbs", "update")); + let deployment_path = artifact_root.join("03-deployments/amber-site-controller.yaml"); let deployment_raw = fs::read_to_string(&deployment_path).expect("deployment should read"); assert!(deployment_raw.contains("amber-site-controller")); @@ -10577,6 +10619,7 @@ spec: fs::read_to_string(artifact_root.join("05-networkpolicies/amber-router-netpol.yaml")) .expect("router netpol should read"); assert!(router_netpol.contains("amber-site-controller")); + assert!(router_netpol.contains("24000")); assert!(router_netpol.contains("24100")); assert!(router_netpol.contains("37046")); let router_service = fs::read_to_string(artifact_root.join("04-services/amber-router.yaml")) @@ -10587,6 +10630,11 @@ spec: artifact_root.join("05-networkpolicies/amber-site-controller-netpol.yaml"), ) .expect("site controller netpol should read"); + assert!( + controller_netpol.contains("24000"), + "site controller needs router mesh egress for in-cluster router readiness probes" + ); + assert!(controller_netpol.contains("24100")); assert!(controller_netpol.contains("37046")); }